TEP-0118: Enable pipeline to handle matrix include params

[TEP-0090: Matrix] introduced `Matrix` to the `PipelineTask` specification such that the `PipelineTask` executes a list of `TaskRuns` or `Runs` in parallel with the specified list of inputs for a `Parameter` or with different combinations of the inputs for a set of `Parameters`.

To build on this, Tep-0018 introduced Matrix.Include, which allows passing in a specific combinations of `Parameters` into the `Matrix`.

This commit enables pipeline to handle matrix include params

Note: This feature is still in preview mode. Implementation logic will be added in subsequent commits.

examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml

@@ -21,7 +21,7 @@ kind: PipelineRun
 metadata:
   generateName: matrixed-pr-
 spec:
-  serviceAccountName: 'default'
+  serviceAccountName: "default"
   pipelineSpec:
     tasks:
       - name: get-platforms

pkg/apis/pipeline/v1/pipeline_types.go

@@ -267,6 +267,29 @@ func (pt *PipelineTask) IsMatrixed() bool {
 	return pt.Matrix != nil && (pt.Matrix.hasParams() || pt.Matrix.hasInclude())
 }
 
+// extractAllParams extracts all the parameter in a Pipeline Task including Matrix parameter
+// and Matrix Include parameter
+func (pt *PipelineTask) extractAllParams() Params {
+	allParams := pt.Params
+	if pt.Matrix.hasParams() {
+		allParams = append(allParams, pt.Matrix.Params...)
+	}
+	if pt.Matrix.hasInclude() {
+		for _, include := range pt.Matrix.Include {
+			allParams = append(allParams, include.Params...)
+		}
+	}
+	return allParams
+}
+
+// extractAllParamValues extracts all the parameter values in a Pipeline Task including Matrix parameter
+// and Matrix Include parameter
+func (pt *PipelineTask) extractAllParamValues() []string {
+	allParams := pt.extractAllParams()
+	paramValues := allParams.extractParamValuesFromParams()
+	return paramValues
+}
+
 func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldError) {
 	if pt.IsMatrixed() {
 		// This is an alpha feature and will fail validation if it's used in a pipeline spec

pkg/apis/pipeline/v1/pipeline_validation.go

@@ -183,26 +183,7 @@ func validatePipelineContextVariables(tasks []PipelineTask) *apis.FieldError {
 	)
 	var paramValues []string
 	for _, task := range tasks {
-		var matrixParams []Param
-		var includeParams []Param
-		if task.IsMatrixed() {
-			matrixParams = task.Matrix.Params
-			if task.Matrix.hasInclude() {
-				for _, include := range task.Matrix.Include {
-					includeParams = include.Params
-				}
-			}
-		}
-		for _, param := range append(task.Params, matrixParams...) {
-			paramValues = append(paramValues, param.Value.StringVal)
-			paramValues = append(paramValues, param.Value.ArrayVal...)
-		}
-
-		if task.Matrix.hasInclude() {
-			for _, param := range append(task.Params, includeParams...) {
-				paramValues = append(paramValues, param.Value.StringVal)
-			}
-		}
+		paramValues = task.extractAllParamValues()
 	}
 	errs := validatePipelineContextVariablesInParamValues(paramValues, "context\\.pipelineRun", pipelineRunContextNames).
 		Also(validatePipelineContextVariablesInParamValues(paramValues, "context\\.pipeline", pipelineContextNames)).

pkg/apis/pipeline/v1/resultref.go

@@ -200,19 +200,13 @@ func ParseResultName(resultName string) (string, string) {
 // in a PipelineTask and returns a list of any references that are found.
 func PipelineTaskResultRefs(pt *PipelineTask) []*ResultRef {
 	refs := []*ResultRef{}
-	var matrixParams []Param
-	if pt.IsMatrixed() {
-		matrixParams = pt.Matrix.Params
-	}
-	for _, p := range append(pt.Params, matrixParams...) {
+	for _, p := range pt.extractAllParams() {
 		expressions, _ := GetVarSubstitutionExpressionsForParam(p)
 		refs = append(refs, NewResultRefs(expressions)...)
 	}
-
 	for _, whenExpression := range pt.When {
 		expressions, _ := whenExpression.GetVarSubstitutionExpressions()
 		refs = append(refs, NewResultRefs(expressions)...)
 	}
-
 	return refs
 }

pkg/apis/pipeline/v1/resultref_test.go

@@ -639,10 +639,16 @@ func TestPipelineTaskResultRefs(t *testing.T) {
 			},
 		}},
 		Matrix: &v1.Matrix{
+			Include: []v1.MatrixInclude{{
+				Name: "build-1",
+				Params: []v1.Param{{
+					Name: "a-param", Value: *v1.NewStructuredValues("$(tasks.pt5.results.r5)"),
+				}},
+			}},
 			Params: []v1.Param{{
-				Value: *v1.NewStructuredValues("$(tasks.pt5.results.r5)", "$(tasks.pt6.results.r6)"),
+				Value: *v1.NewStructuredValues("$(tasks.pt6.results.r6)", "$(tasks.pt7.results.r7)"),
 			}, {
-				Value: *v1.NewStructuredValues("$(tasks.pt7.results.r7)", "$(tasks.pt8.results.r8)"),
+				Value: *v1.NewStructuredValues("$(tasks.pt8.results.r8)", "$(tasks.pt9.results.r9)"),
 			}}},
 	}
 	refs := v1.PipelineTaskResultRefs(&pt)
@@ -670,6 +676,9 @@ func TestPipelineTaskResultRefs(t *testing.T) {
 	}, {
 		PipelineTask: "pt8",
 		Result:       "r8",
+	}, {
+		PipelineTask: "pt9",
+		Result:       "r9",
 	}}
 	if d := cmp.Diff(refs, expectedRefs, cmpopts.SortSlices(lessResultRef)); d != "" {
 		t.Errorf("%v", d)

pkg/apis/pipeline/v1beta1/pipeline_conversion.go

@@ -259,6 +259,16 @@ func (m *Matrix) convertTo(ctx context.Context, sink *v1.Matrix) {
 		param.convertTo(ctx, &new)
 		sink.Params = append(sink.Params, new)
 	}
+	if m.hasInclude() {
+		for _, include := range m.Include {
+			sink.Include = append(sink.Include, v1.MatrixInclude{Name: include.Name})
+			for _, param := range include.Params {
+				newIncludeParam := v1.Param{}
+				param.convertTo(ctx, &newIncludeParam)
+				sink.Include[len(sink.Include)-1].Params = append(sink.Include[len(sink.Include)-1].Params, newIncludeParam)
+			}
+		}
+	}
 }
 
 func (m *Matrix) convertFrom(ctx context.Context, source v1.Matrix) {
@@ -267,6 +277,15 @@ func (m *Matrix) convertFrom(ctx context.Context, source v1.Matrix) {
 		new.convertFrom(ctx, param)
 		m.Params = append(m.Params, new)
 	}
+
+	for _, include := range source.Include {
+		m.Include = append(m.Include, MatrixInclude{Name: include.Name})
+		for _, p := range include.Params {
+			new := Param{}
+			new.convertFrom(ctx, p)
+			m.Include[len(m.Include)-1].Params = append(m.Include[len(m.Include)-1].Params, new)
+		}
+	}
 }
 
 func (pr PipelineResult) convertTo(ctx context.Context, sink *v1.PipelineResult) {

pkg/apis/pipeline/v1beta1/pipeline_conversion_test.go

@@ -108,13 +108,21 @@ func TestPipelineConversion(t *testing.T) {
 						},
 					}},
 					Matrix: &v1beta1.Matrix{
-						Params: []v1beta1.Param{{
+						Params: v1beta1.Params{{
 							Name: "a-param",
 							Value: v1beta1.ParamValue{
 								Type:     v1beta1.ParamTypeArray,
 								ArrayVal: []string{"$(params.baz)", "and", "$(params.foo-is-baz)"},
 							},
-						}}},
+						}},
+						Include: []v1beta1.MatrixInclude{{
+							Name: "baz",
+							Params: v1beta1.Params{{
+								Name: "a-param", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeString, StringVal: "$(params.baz)"},
+							}, {
+								Name: "flags", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeString, StringVal: "-cover -v"}}},
+						}},
+					},
 					Workspaces: []v1beta1.WorkspacePipelineTaskBinding{{
 						Name:      "my-task-workspace",
 						Workspace: "source",

pkg/apis/pipeline/v1beta1/pipeline_types.go

@@ -283,6 +283,29 @@ func (pt *PipelineTask) IsMatrixed() bool {
 	return pt.Matrix != nil && (pt.Matrix.hasParams() || pt.Matrix.hasInclude())
 }
 
+// extractAllParams extracts all the params in a Pipeline Task including Matrix params
+// and Matrix Include params
+func (pt *PipelineTask) extractAllParams() Params {
+	allParams := pt.Params
+	if pt.Matrix.hasParams() {
+		allParams = append(allParams, pt.Matrix.Params...)
+	}
+	if pt.Matrix.hasInclude() {
+		for _, include := range pt.Matrix.Include {
+			allParams = append(allParams, include.Params...)
+		}
+	}
+	return allParams
+}
+
+// extractAllParamValues extracts all the parameter values in a Pipeline Task including Matrix parameter
+// and Matrix Include parameter
+func (pt *PipelineTask) extractAllParamValues() []string {
+	allParams := pt.extractAllParams()
+	paramValues := allParams.extractParamValuesFromParams()
+	return paramValues
+}
+
 func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldError) {
 	if pt.IsMatrixed() {
 		// This is an alpha feature and will fail validation if it's used in a pipeline spec

pkg/apis/pipeline/v1beta1/pipeline_validation.go

@@ -183,26 +183,7 @@ func validatePipelineContextVariables(tasks []PipelineTask) *apis.FieldError {
 	)
 	var paramValues []string
 	for _, task := range tasks {
-		var matrixParams []Param
-		var includeParams []Param
-		if task.IsMatrixed() {
-			matrixParams = task.Matrix.Params
-			if task.Matrix.hasInclude() {
-				for _, include := range task.Matrix.Include {
-					includeParams = include.Params
-				}
-			}
-		}
-		for _, param := range append(task.Params, matrixParams...) {
-			paramValues = append(paramValues, param.Value.StringVal)
-			paramValues = append(paramValues, param.Value.ArrayVal...)
-		}
-
-		if task.Matrix.hasInclude() {
-			for _, param := range append(task.Params, includeParams...) {
-				paramValues = append(paramValues, param.Value.StringVal)
-			}
-		}
+		paramValues = task.extractAllParamValues()
 	}
 	errs := validatePipelineContextVariablesInParamValues(paramValues, "context\\.pipelineRun", pipelineRunContextNames).
 		Also(validatePipelineContextVariablesInParamValues(paramValues, "context\\.pipeline", pipelineContextNames)).

pkg/apis/pipeline/v1beta1/resultref.go

@@ -208,19 +208,13 @@ func ParseResultName(resultName string) (string, string) {
 // in a PipelineTask and returns a list of any references that are found.
 func PipelineTaskResultRefs(pt *PipelineTask) []*ResultRef {
 	refs := []*ResultRef{}
-	var matrixParams []Param
-	if pt.IsMatrixed() {
-		matrixParams = pt.Matrix.Params
-	}
-	for _, p := range append(pt.Params, matrixParams...) {
+	for _, p := range pt.extractAllParams() {
 		expressions, _ := GetVarSubstitutionExpressionsForParam(p)
 		refs = append(refs, NewResultRefs(expressions)...)
 	}
-
 	for _, whenExpression := range pt.WhenExpressions {
 		expressions, _ := whenExpression.GetVarSubstitutionExpressions()
 		refs = append(refs, NewResultRefs(expressions)...)
 	}
-
 	return refs
 }

pkg/apis/pipeline/v1beta1/resultref_test.go

@@ -639,10 +639,16 @@ func TestPipelineTaskResultRefs(t *testing.T) {
 			},
 		}},
 		Matrix: &v1beta1.Matrix{
+			Include: []v1beta1.MatrixInclude{{
+				Name: "build-1",
+				Params: v1beta1.Params{{
+					Name: "a-param", Value: *v1beta1.NewStructuredValues("$(tasks.pt5.results.r5)"),
+				}},
+			}},
 			Params: []v1beta1.Param{{
-				Value: *v1beta1.NewStructuredValues("$(tasks.pt5.results.r5)", "$(tasks.pt6.results.r6)"),
+				Value: *v1beta1.NewStructuredValues("$(tasks.pt6.results.r6)", "$(tasks.pt7.results.r7)"),
 			}, {
-				Value: *v1beta1.NewStructuredValues("$(tasks.pt7.results.r7)", "$(tasks.pt8.results.r8)"),
+				Value: *v1beta1.NewStructuredValues("$(tasks.pt8.results.r8)", "$(tasks.pt9.results.r9)"),
 			}}},
 	}
 	refs := v1beta1.PipelineTaskResultRefs(&pt)
@@ -670,6 +676,9 @@ func TestPipelineTaskResultRefs(t *testing.T) {
 	}, {
 		PipelineTask: "pt8",
 		Result:       "r8",
+	}, {
+		PipelineTask: "pt9",
+		Result:       "r9",
 	}}
 	if d := cmp.Diff(refs, expectedRefs, cmpopts.SortSlices(lessResultRef)); d != "" {
 		t.Errorf("%v", d)

pkg/reconciler/pipelinerun/resources/apply.go

@@ -176,6 +176,9 @@ func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResul
 			pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements, objectReplacements)
 			if pipelineTask.IsMatrixed() {
 				pipelineTask.Matrix.Params = replaceParamValues(pipelineTask.Matrix.Params, stringReplacements, nil, nil)
+				for i := range pipelineTask.Matrix.Include {
+					pipelineTask.Matrix.Include[i].Params = replaceParamValues(pipelineTask.Matrix.Include[i].Params, stringReplacements, nil, nil)
+				}
 			}
 			pipelineTask.WhenExpressions = pipelineTask.WhenExpressions.ReplaceWhenExpressionsVariables(stringReplacements, arrayReplacements)
 			if pipelineTask.TaskRef != nil && pipelineTask.TaskRef.Params != nil {
@@ -225,6 +228,9 @@ func ApplyReplacements(p *v1beta1.PipelineSpec, replacements map[string]string,
 		p.Tasks[i].Params = replaceParamValues(p.Tasks[i].Params, replacements, arrayReplacements, objectReplacements)
 		if p.Tasks[i].IsMatrixed() {
 			p.Tasks[i].Matrix.Params = replaceParamValues(p.Tasks[i].Matrix.Params, replacements, arrayReplacements, objectReplacements)
+			for j := range p.Tasks[i].Matrix.Include {
+				p.Tasks[i].Matrix.Include[j].Params = replaceParamValues(p.Tasks[i].Matrix.Include[j].Params, replacements, arrayReplacements, objectReplacements)
+			}
 		}
 		for j := range p.Tasks[i].Workspaces {
 			p.Tasks[i].Workspaces[j].SubPath = substitution.ApplyReplacements(p.Tasks[i].Workspaces[j].SubPath, replacements)
@@ -240,6 +246,9 @@ func ApplyReplacements(p *v1beta1.PipelineSpec, replacements map[string]string,
 		p.Finally[i].Params = replaceParamValues(p.Finally[i].Params, replacements, arrayReplacements, objectReplacements)
 		if p.Finally[i].IsMatrixed() {
 			p.Finally[i].Matrix.Params = replaceParamValues(p.Finally[i].Matrix.Params, replacements, arrayReplacements, objectReplacements)
+			for j := range p.Finally[i].Matrix.Include {
+				p.Finally[i].Matrix.Include[j].Params = replaceParamValues(p.Finally[i].Matrix.Include[j].Params, replacements, arrayReplacements, objectReplacements)
+			}
 		}
 		for j := range p.Finally[i].Workspaces {
 			p.Finally[i].Workspaces[j].SubPath = substitution.ApplyReplacements(p.Finally[i].Workspaces[j].SubPath, replacements)