Skip to content

Commit

Permalink
simplifying matrix combinations
Browse files Browse the repository at this point in the history 
Matrix combinations is created in [][]Param format which is very hard
to process with matrix.Include. The parameters value can change in a combination
or a new combination can be added based on matrix.Include.Params.

Its easy to compare and work with map[string]string instead of []Param.

Signed-off-by: pritidesai <pdesai@us.ibm.com>
  • Loading branch information
@pritidesai
pritidesai committed Mar 8, 2023
1 parent e8f2637 commit 7f695cb
Show file tree
Hide file tree
Showing 8 changed files with 148 additions and 139 deletions.
25 changes: 13 additions & 12 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,25 @@ require github.com/benbjohnson/clock v1.1.0 // indirect

require (
code.gitea.io/sdk/gitea v0.15.1
github.com/cenkalti/backoff/v3 v3.2.2
github.com/goccy/kpoward v0.1.0
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221030203717-1711cefd7eec
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-retryablehttp v0.7.2
github.com/hashicorp/go-rootcerts v1.0.2
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2
github.com/hashicorp/go-sockaddr v1.0.2
github.com/hashicorp/hcl v1.0.0
github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf
github.com/mitchellh/mapstructure v1.5.0
github.com/ryanuber/go-glob v1.0.0
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399
go.opentelemetry.io/otel v1.14.0
go.opentelemetry.io/otel/exporters/jaeger v1.14.0
go.opentelemetry.io/otel/sdk v1.14.0
go.opentelemetry.io/otel/trace v1.14.0
golang.org/x/exp v0.0.0-20230307190834-24139beb5833
k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2
)

Expand All @@ -73,7 +84,6 @@ require (
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/aws/aws-sdk-go-v2/service/kms v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/cloudflare/circl v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/fatih/color v1.13.0 // indirect
Expand All @@ -83,21 +93,12 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/vault/api v1.9.0 // indirect
github.com/jellydator/ttlcache/v2 v2.11.1 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/skeema/knownhosts v1.1.0 // indirect
github.com/theupdateframework/go-tuf v0.5.2 // indirect
github.com/zeebo/errs v1.3.0 // indirect
Expand Down Expand Up @@ -193,12 +194,12 @@ require (
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.6.0
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/net v0.8.0
golang.org/x/sync v0.1.0
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/time v0.3.0
golang.org/x/tools v0.6.0 // indirect
google.golang.org/api v0.110.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

30 changes: 13 additions & 17 deletions pkg/apis/pipeline/v1/matrix_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"fmt"

"github.com/tektoncd/pipeline/pkg/apis/config"
"golang.org/x/exp/maps"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/strings/slices"
"knative.dev/pkg/apis"
Expand Down Expand Up @@ -52,50 +53,45 @@ type MatrixInclude struct {
}

// FanOut produces combinations of Parameters of type String from a slice of Parameters of type Array.
func (m *Matrix) FanOut() []Params {
var combinations []Params
func (m *Matrix) FanOut() []map[string]string {
var combinations []map[string]string
for _, parameter := range m.Params {
combinations = fanOut(parameter, combinations)
}
return combinations
}

// fanOut generates a new combination based on a given Parameter in the Matrix.
func fanOut(param Param, combinations []Params) []Params {
func fanOut(param Param, combinations []map[string]string) []map[string]string {
if len(combinations) == 0 {
return initializeCombinations(param)
}
return distribute(param, combinations)
}

// distribute generates a new combination of Parameters by adding a new Parameter to an existing list of Combinations.
func distribute(param Param, combinations []Params) []Params {
var expandedCombinations []Params
func distribute(param Param, combinations []map[string]string) []map[string]string {
var expandedCombinations []map[string]string
for _, value := range param.Value.ArrayVal {
for _, combination := range combinations {
expandedCombinations = append(expandedCombinations, createCombination(param.Name, value, combination))
newCombination := make(map[string]string)
maps.Copy(newCombination, combination)
newCombination[param.Name] = value
expandedCombinations = append(expandedCombinations, newCombination)
}
}
return expandedCombinations
}

// initializeCombinations generates a new combination based on the first Parameter in the Matrix.
func initializeCombinations(param Param) []Params {
var combinations []Params
func initializeCombinations(param Param) []map[string]string {
var combinations []map[string]string
for _, value := range param.Value.ArrayVal {
combinations = append(combinations, createCombination(param.Name, value, []Param{}))
combinations = append(combinations, map[string]string{param.Name: value})
}
return combinations
}

func createCombination(name string, value string, combination Params) Params {
combination = append(combination, Param{
Name: name,
Value: ParamValue{Type: ParamTypeString, StringVal: value},
})
return combination
}

// CountCombinations returns the count of combinations of Parameters generated from the Matrix in PipelineTask.
func (m *Matrix) CountCombinations() int {
// Iterate over matrix.params and compute count of all generated combinations
Expand Down
102 changes: 24 additions & 78 deletions pkg/apis/pipeline/v1/matrix_types_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestMatrix_FanOut(t *testing.T) {
tests := []struct {
name string
matrix Matrix
want []Params
want []map[string]string
}{{
name: "matrix with no params",
matrix: Matrix{
Expand All @@ -39,21 +39,12 @@ func TestMatrix_FanOut(t *testing.T) {
Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"linux", "mac", "windows"}},
}},
},
want: []Params{{
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "linux"},
},
want: []map[string]string{{
"platform": "linux",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "mac"},
},
"platform": "mac",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "windows"},
},
"platform": "windows",
}},
}, {
name: "multiple arrays in matrix",
Expand All @@ -65,78 +56,33 @@ func TestMatrix_FanOut(t *testing.T) {
Name: "browser",
Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}},
}}},
want: []Params{{
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "linux"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "chrome"},
},
want: []map[string]string{{
"platform": "linux",
"browser": "chrome",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "mac"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "chrome"},
},
"platform": "mac",
"browser": "chrome",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "windows"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "chrome"},
},
"platform": "windows",
"browser": "chrome",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "linux"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "safari"},
},
"platform": "linux",
"browser": "safari",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "mac"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "safari"},
},
"platform": "mac",
"browser": "safari",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "windows"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "safari"},
},
"platform": "windows",
"browser": "safari",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "linux"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "firefox"},
},
"platform": "linux",
"browser": "firefox",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "mac"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "firefox"},
},
"platform": "mac",
"browser": "firefox",
}, {
{
Name: "platform",
Value: ParamValue{Type: ParamTypeString, StringVal: "windows"},
}, {
Name: "browser",
Value: ParamValue{Type: ParamTypeString, StringVal: "firefox"},
},
"platform": "windows",
"browser": "firefox",
}},
}}
for _, tt := range tests {
Expand Down
63 changes: 40 additions & 23 deletions pkg/apis/pipeline/v1beta1/matrix_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package v1beta1
import (
"context"
"fmt"
"golang.org/x/exp/maps"

"github.com/tektoncd/pipeline/pkg/apis/config"
"k8s.io/apimachinery/pkg/util/sets"
Expand All @@ -37,6 +38,11 @@ type Matrix struct {
// +optional
// +listType=atomic
Include []MatrixInclude `json:"include,omitempty"`

// combinations is a list of combinations created based on a given Matrix configurations including
// Matrix.Params and Matrix.Include.Params
// Combinations is skipped while encoding JSON since they are calculated based on the specified params
Combinations Combinations `json:"-"`
}

// MatrixInclude allows passing in a specific combinations of Parameters into the Matrix.
Expand All @@ -51,51 +57,62 @@ type MatrixInclude struct {
Params Params `json:"params,omitempty"`
}

type Combination map[string]string
type Combinations []Combination

// FanOut produces combinations of Parameters of type String from a slice of Parameters of type Array.
func (m *Matrix) FanOut() []Params {
var combinations []Params
func (m *Matrix) FanOut() {
var combinations Combinations
for _, parameter := range m.Params {
combinations = fanOut(parameter, combinations)
combinations = combinations.fanOut(parameter)
}
return combinations
m.Combinations = combinations
}

func (c Combinations) FormatCombinations() []Params {
ps := make([]Params, len(c))
for i := range c {
for k, v := range c[i] {
ps[i] = append(ps[i], Param{
Name: k,
Value: ParamValue{Type: ParamTypeString, StringVal: v},
})
}
}
return ps
}

// fanOut generates a new combination based on a given Parameter in the Matrix.
func fanOut(param Param, combinations []Params) []Params {
if len(combinations) == 0 {
return initializeCombinations(param)
func (c Combinations) fanOut(param Param) Combinations {
if len(c) == 0 {
return c.initializeCombinations(param)
}
return distribute(param, combinations)
return c.distribute(param)
}

// distribute generates a new combination of Parameters by adding a new Parameter to an existing list of Combinations.
func distribute(param Param, combinations []Params) []Params {
var expandedCombinations []Params
func (c Combinations) distribute(param Param) Combinations {
var expandedCombinations Combinations
for _, value := range param.Value.ArrayVal {
for _, combination := range combinations {
expandedCombinations = append(expandedCombinations, createCombination(param.Name, value, combination))
for _, combination := range c {
newCombination := make(map[string]string)
maps.Copy(newCombination, combination)
newCombination[param.Name] = value
expandedCombinations = append(expandedCombinations, newCombination)
}
}
return expandedCombinations
}

// initializeCombinations generates a new combination based on the first Parameter in the Matrix.
func initializeCombinations(param Param) []Params {
var combinations []Params
func (c Combinations) initializeCombinations(param Param) Combinations {
var combinations Combinations
for _, value := range param.Value.ArrayVal {
combinations = append(combinations, createCombination(param.Name, value, []Param{}))
combinations = append(combinations, Combination{param.Name: value})
}
return combinations
}

func createCombination(name string, value string, combination Params) Params {
combination = append(combination, Param{
Name: name,
Value: ParamValue{Type: ParamTypeString, StringVal: value},
})
return combination
}

// CountCombinations returns the count of combinations of Parameters generated from the Matrix in PipelineTask.
func (m *Matrix) CountCombinations() int {
// Iterate over matrix.params and compute count of all generated combinations
Expand Down
Loading

0 comments on commit 7f695cb

Please sign in to comment.