From b0184571534b975efa8d8f96eff5f727df86ed3e Mon Sep 17 00:00:00 2001 From: Emma Munley Date: Fri, 10 Mar 2023 13:08:49 -0500 Subject: [PATCH] TEP-0118: Apply Replacements for Matrix Include Parameters This commit replaces Matrix Include Parameters with specified values. Note: This feature is still in preview mode. --- pkg/reconciler/pipelinerun/resources/apply.go | 11 + .../pipelinerun/resources/apply_test.go | 421 ++++++++++++++++++ 2 files changed, 432 insertions(+) diff --git a/pkg/reconciler/pipelinerun/resources/apply.go b/pkg/reconciler/pipelinerun/resources/apply.go index 470485a30fc..4721d3e597e 100644 --- a/pkg/reconciler/pipelinerun/resources/apply.go +++ b/pkg/reconciler/pipelinerun/resources/apply.go @@ -175,7 +175,12 @@ func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResul pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy() pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements, objectReplacements) if pipelineTask.IsMatrixed() { + // Note: Only string results for matrix parameters are supported at this time pipelineTask.Matrix.Params = replaceParamValues(pipelineTask.Matrix.Params, stringReplacements, nil, nil) + for i := range pipelineTask.Matrix.Include { + // matrix include parameters can only be type string + pipelineTask.Matrix.Include[i].Params = replaceParamValues(pipelineTask.Matrix.Include[i].Params, stringReplacements, nil, nil) + } } pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements) if pipelineTask.TaskRef != nil && pipelineTask.TaskRef.Params != nil { @@ -225,6 +230,9 @@ func ApplyReplacements(p *v1beta1.PipelineSpec, replacements map[string]string, p.Tasks[i].Params = replaceParamValues(p.Tasks[i].Params, replacements, arrayReplacements, objectReplacements) if p.Tasks[i].IsMatrixed() { p.Tasks[i].Matrix.Params = replaceParamValues(p.Tasks[i].Matrix.Params, replacements, arrayReplacements, objectReplacements) + for j := range p.Tasks[i].Matrix.Include { + p.Tasks[i].Matrix.Include[j].Params = replaceParamValues(p.Tasks[i].Matrix.Include[j].Params, replacements, arrayReplacements, objectReplacements) + } } for j := range p.Tasks[i].Workspaces { p.Tasks[i].Workspaces[j].SubPath = substitution.ApplyReplacements(p.Tasks[i].Workspaces[j].SubPath, replacements) @@ -240,6 +248,9 @@ func ApplyReplacements(p *v1beta1.PipelineSpec, replacements map[string]string, p.Finally[i].Params = replaceParamValues(p.Finally[i].Params, replacements, arrayReplacements, objectReplacements) if p.Finally[i].IsMatrixed() { p.Finally[i].Matrix.Params = replaceParamValues(p.Finally[i].Matrix.Params, replacements, arrayReplacements, objectReplacements) + for j := range p.Finally[i].Matrix.Include { + p.Finally[i].Matrix.Include[j].Params = replaceParamValues(p.Finally[i].Matrix.Include[j].Params, replacements, arrayReplacements, objectReplacements) + } } for j := range p.Finally[i].Workspaces { p.Finally[i].Workspaces[j].SubPath = substitution.ApplyReplacements(p.Finally[i].Workspaces[j].SubPath, replacements) diff --git a/pkg/reconciler/pipelinerun/resources/apply_test.go b/pkg/reconciler/pipelinerun/resources/apply_test.go index 0bdaa3c830a..0ff283ab859 100644 --- a/pkg/reconciler/pipelinerun/resources/apply_test.go +++ b/pkg/reconciler/pipelinerun/resources/apply_test.go @@ -2034,6 +2034,285 @@ func TestApplyParameters_ArrayIndexing(t *testing.T) { } } +func TestApplyReplacementsMatrix(t *testing.T) { + ctx := context.Background() + cfg := config.FromContextOrDefaults(ctx) + cfg.FeatureFlags.EnableAPIFields = config.BetaAPIFields + ctx = config.ToContext(ctx, cfg) + for _, tt := range []struct { + name string + original v1beta1.PipelineSpec + params []v1beta1.Param + expected v1beta1.PipelineSpec + }{{ + name: "matrix params - array replacement", + original: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "first-param", Type: v1beta1.ParamTypeArray, Default: v1beta1.NewStructuredValues("default", "array", "value"), + }, { + Name: "second-param", Type: v1beta1.ParamTypeArray, + }}, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Params: []v1beta1.Param{{ + Name: "first-matrix-task-first-param", Value: *v1beta1.NewStructuredValues("firstelement", "$(params.first-param)"), + }, { + Name: "first-matrix-task-second-param", Value: *v1beta1.NewStructuredValues("firstelement", "$(params.second-param[0])"), + }}, + }, + }}, + }, + params: []v1beta1.Param{ + {Name: "second-param", Value: *v1beta1.NewStructuredValues("second-value", "array")}, + }, + expected: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{ + {Name: "first-param", Type: v1beta1.ParamTypeArray, Default: v1beta1.NewStructuredValues("default", "array", "value")}, + {Name: "second-param", Type: v1beta1.ParamTypeArray}, + }, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Params: []v1beta1.Param{ + {Name: "first-matrix-task-first-param", Value: *v1beta1.NewStructuredValues("firstelement", "default", "array", "value")}, + {Name: "first-matrix-task-second-param", Value: *v1beta1.NewStructuredValues("firstelement", "second-value")}, + }, + }, + }}, + }, + }, { + name: "matrix include params - string replacement", + original: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "first-param", Type: v1beta1.ParamTypeString, Default: v1beta1.NewStructuredValues("default-value"), + }}, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "first-param", Value: *v1beta1.NewStructuredValues("$(params.first-param)"), + }}}, + }}, + }}, + }, + params: v1beta1.Params{{Name: "first-param", Value: *v1beta1.NewStructuredValues("default-value")}}, + expected: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "first-param", Type: v1beta1.ParamTypeString, Default: v1beta1.NewStructuredValues("default-value"), + }}, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "first-param", Value: *v1beta1.NewStructuredValues("default-value"), + }}}, + }}, + }}, + }, + }, { + name: "matrix include params - string replacement - override default", + original: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "first-param", Type: v1beta1.ParamTypeString, Default: v1beta1.NewStructuredValues("default-value"), + }, { + Name: "second-param", Type: v1beta1.ParamTypeString}}, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "first-param", Value: *v1beta1.NewStructuredValues("$(params.first-param)"), + }, { + Name: "second-param", Value: *v1beta1.NewStructuredValues("$(params.second-param)"), + }, { + Name: "third-param", Value: *v1beta1.NewStructuredValues("new-value"), + }}}, + }}, + }}, + }, + params: v1beta1.Params{{Name: "second-param", Value: *v1beta1.NewStructuredValues("second-value")}}, + expected: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "first-param", Type: v1beta1.ParamTypeString, Default: v1beta1.NewStructuredValues("default-value"), + }, { + Name: "second-param", Type: v1beta1.ParamTypeString, + }}, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "first-param", Value: *v1beta1.NewStructuredValues("default-value"), + }, { + Name: "second-param", Value: *v1beta1.NewStructuredValues("second-value"), + }, { + Name: "third-param", Value: *v1beta1.NewStructuredValues("new-value"), + }}}, + }}, + }}, + }, + }, { + name: "matrix include params - object replacement", + original: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "myobject", + Properties: map[string]v1beta1.PropertySpec{ + "key1": {Type: "string"}, + "key2": {Type: "string"}, + }, + Default: v1beta1.NewObject(map[string]string{ + "key1": "default", + "key2": "param!", + }), + }}, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "first-param", Value: *v1beta1.NewStructuredValues("$(params.myobject.key1)"), + }, { + Name: "second-param", Value: *v1beta1.NewStructuredValues("$(params.myobject.key2)"), + }}}, + }}, + }}, + }, + params: v1beta1.Params{{Name: "myobject", Value: *v1beta1.NewObject(map[string]string{ + "key1": "pipeline", + "key2": "param!!", + })}}, + expected: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{{ + Name: "myobject", + Properties: map[string]v1beta1.PropertySpec{ + "key1": {Type: "string"}, + "key2": {Type: "string"}, + }, + Default: v1beta1.NewObject(map[string]string{ + "key1": "default", + "key2": "param!", + }), + }}, + Tasks: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "first-param", Value: *v1beta1.NewStructuredValues("pipeline"), + }, { + Name: "second-param", Value: *v1beta1.NewStructuredValues("param!!"), + }}, + }, + }}, + }}, + }, + }, { + name: "matrix params with final tasks", + original: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{ + {Name: "first-param", Type: v1beta1.ParamTypeString, Default: v1beta1.NewStructuredValues("default-value")}, + {Name: "second-param", Type: v1beta1.ParamTypeString}, + }, + Finally: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Params: v1beta1.Params{{ + Name: "final-task-first-param", Value: *v1beta1.NewStructuredValues("$(params.first-param)"), + }, { + Name: "final-task-second-param", Value: *v1beta1.NewStructuredValues("$(params.second-param)"), + }}}, + WhenExpressions: v1beta1.WhenExpressions{{ + Input: "$(params.first-param)", + Operator: selection.In, + Values: []string{"$(params.second-param)"}, + }}, + }}, + }, + params: v1beta1.Params{{Name: "second-param", Value: *v1beta1.NewStructuredValues("second-value")}}, + expected: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{ + {Name: "first-param", Type: v1beta1.ParamTypeString, Default: v1beta1.NewStructuredValues("default-value")}, + {Name: "second-param", Type: v1beta1.ParamTypeString}, + }, + Finally: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Params: v1beta1.Params{{ + Name: "final-task-first-param", Value: *v1beta1.NewStructuredValues("default-value"), + }, { + Name: "final-task-second-param", Value: *v1beta1.NewStructuredValues("second-value"), + }}}, + WhenExpressions: v1beta1.WhenExpressions{{ + Input: "default-value", + Operator: selection.In, + Values: []string{"second-value"}, + }}, + }}, + }, + }, { + name: "matrix include params with final tasks", + original: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{ + {Name: "first-param", Type: v1beta1.ParamTypeArray, Default: v1beta1.NewStructuredValues("default-value", "default-value-again")}, + {Name: "second-param", Type: v1beta1.ParamTypeArray}, + }, + Finally: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "final-task-first-param", Value: *v1beta1.NewStructuredValues("$(params.first-param[0])"), + }, { + Name: "final-task-second-param", Value: *v1beta1.NewStructuredValues("$(params.second-param[1])"), + }}}, + }}, + WhenExpressions: v1beta1.WhenExpressions{{ + Input: "$(params.first-param[0])", + Operator: selection.In, + Values: []string{"$(params.second-param[1])"}, + }}, + }}, + }, + params: v1beta1.Params{{Name: "second-param", Value: *v1beta1.NewStructuredValues("second-value", "second-value-again")}}, + expected: v1beta1.PipelineSpec{ + Params: []v1beta1.ParamSpec{ + {Name: "first-param", Type: v1beta1.ParamTypeArray, Default: v1beta1.NewStructuredValues("default-value", "default-value-again")}, + {Name: "second-param", Type: v1beta1.ParamTypeArray}, + }, + Finally: []v1beta1.PipelineTask{{ + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: v1beta1.Params{{ + Name: "final-task-first-param", Value: *v1beta1.NewStructuredValues("default-value"), + }, { + Name: "final-task-second-param", Value: *v1beta1.NewStructuredValues("second-value-again"), + }}}, + }}, + WhenExpressions: v1beta1.WhenExpressions{{ + Input: "default-value", + Operator: selection.In, + Values: []string{"second-value-again"}, + }}, + }}, + }, + }, + } { + tt := tt // capture range variable + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + run := &v1beta1.PipelineRun{ + Spec: v1beta1.PipelineRunSpec{ + Params: tt.params, + }, + } + got := ApplyParameters(ctx, &tt.original, run) + if d := cmp.Diff(&tt.expected, got); d != "" { + t.Errorf("ApplyParameters() got diff %s", diff.PrintWantGot(d)) + } + }) + } +} + func TestApplyTaskResults_MinimalExpression(t *testing.T) { for _, tt := range []struct { name string @@ -2330,6 +2609,76 @@ func TestApplyTaskResults_MinimalExpression(t *testing.T) { }}}, }, }}, + }, { + name: "Test result substitution on minimal variable substitution expression - matrix include", + resolvedResultRefs: ResolvedResultRefs{{ + Value: *v1beta1.NewStructuredValues("aResultValue"), + ResultReference: v1beta1.ResultRef{ + PipelineTask: "aTask", + Result: "a.Result", + }, + FromTaskRun: "aTaskRun", + }}, + targets: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", Value: *v1beta1.NewStructuredValues(`$(tasks.aTask.results["a.Result"])`)}}, + }}}, + }, + }}, + want: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", Value: *v1beta1.NewStructuredValues("aResultValue")}}, + }}}, + }, + }}, + }, { + name: "Test array indexing result substitution on minimal variable substitution expression - matrix include", + resolvedResultRefs: ResolvedResultRefs{{ + Value: *v1beta1.NewStructuredValues("arrayResultValueOne", "arrayResultValueTwo"), + ResultReference: v1beta1.ResultRef{ + PipelineTask: "aTask", + Result: "a.Result", + }, + FromTaskRun: "aTaskRun", + }}, + targets: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", + Value: *v1beta1.NewStructuredValues(`$(tasks.aTask.results["a.Result"][1])`)}}, + }}}, + }, + }}, + want: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", + Value: *v1beta1.NewStructuredValues("arrayResultValueTwo")}}, + }}}, + }, + }}, }, { name: "Test array result substitution on minimal variable substitution expression - when expressions", resolvedResultRefs: ResolvedResultRefs{{ @@ -2679,6 +3028,78 @@ func TestApplyTaskResults_EmbeddedExpression(t *testing.T) { }}}, }, }}, + }, { + name: "Test result substitution on embedded variable substitution expression - matrix include", + resolvedResultRefs: ResolvedResultRefs{{ + Value: *v1beta1.NewStructuredValues("aResultValue"), + ResultReference: v1beta1.ResultRef{ + PipelineTask: "aTask", + Result: "aResult", + }, + FromTaskRun: "aTaskRun", + }}, + targets: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", + Value: *v1beta1.NewStructuredValues("Result value --> $(tasks.aTask.results.aResult)")}}, + }}}, + }, + }}, + want: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", + Value: *v1beta1.NewStructuredValues("Result value --> aResultValue")}}, + }}}, + }, + }}, + }, { + name: "Test array indexing result substitution on embedded variable substitution expression - matrix include", + resolvedResultRefs: ResolvedResultRefs{{ + Value: *v1beta1.NewStructuredValues("arrayResultValueOne", "arrayResultValueTwo"), + ResultReference: v1beta1.ResultRef{ + PipelineTask: "aTask", + Result: "aResult", + }, + FromTaskRun: "aTaskRun", + }}, + targets: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", + Value: *v1beta1.NewStructuredValues("Result value --> $(tasks.aTask.results.aResult[0])")}}, + }}}, + }, + }}, + want: PipelineRunState{{ + PipelineTask: &v1beta1.PipelineTask{ + Name: "bTask", + TaskRef: &v1beta1.TaskRef{Name: "bTask"}, + Matrix: &v1beta1.Matrix{ + Include: []v1beta1.MatrixInclude{{ + Name: "build-1", + Params: []v1beta1.Param{{ + Name: "bParam", + Value: *v1beta1.NewStructuredValues("Result value --> arrayResultValueOne")}}, + }}}, + }, + }}, }, { name: "Test result substitution on embedded variable substitution expression - when expressions", resolvedResultRefs: ResolvedResultRefs{{