Skip to content

Commit

Permalink
simplifying matrix combinations
Browse files Browse the repository at this point in the history 
Matrix combinations is created in [][]Param format which is very hard
to process with matrix.Include. The parameters value can change in a combination
or a new combination can be added based on matrix.Include.Params.

Its easy to compare and work with map[string]string instead of []Param.

Signed-off-by: pritidesai <pdesai@us.ibm.com>
  • Loading branch information
@pritidesai
pritidesai committed Mar 9, 2023
1 parent e8f2637 commit 70949c5
Show file tree
Hide file tree
Showing 9 changed files with 374 additions and 184 deletions.
25 changes: 13 additions & 12 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,25 @@ require github.com/benbjohnson/clock v1.1.0 // indirect

require (
code.gitea.io/sdk/gitea v0.15.1
github.com/cenkalti/backoff/v3 v3.2.2
github.com/goccy/kpoward v0.1.0
github.com/google/go-containerregistry/pkg/authn/k8schain v0.0.0-20221030203717-1711cefd7eec
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-retryablehttp v0.7.2
github.com/hashicorp/go-rootcerts v1.0.2
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2
github.com/hashicorp/go-sockaddr v1.0.2
github.com/hashicorp/hcl v1.0.0
github.com/letsencrypt/boulder v0.0.0-20221109233200-85aa52084eaf
github.com/mitchellh/mapstructure v1.5.0
github.com/ryanuber/go-glob v1.0.0
github.com/titanous/rocacheck v0.0.0-20171023193734-afe73141d399
go.opentelemetry.io/otel v1.14.0
go.opentelemetry.io/otel/exporters/jaeger v1.14.0
go.opentelemetry.io/otel/sdk v1.14.0
go.opentelemetry.io/otel/trace v1.14.0
golang.org/x/exp v0.0.0-20230307190834-24139beb5833
k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2
)

Expand All @@ -73,7 +84,6 @@ require (
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/aws/aws-sdk-go-v2/service/kms v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.4 // indirect
github.com/cenkalti/backoff/v3 v3.2.2 // indirect
github.com/cloudflare/circl v1.1.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/fatih/color v1.13.0 // indirect
Expand All @@ -83,21 +93,12 @@ require (
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
github.com/hashicorp/go-rootcerts v1.0.2 // indirect
github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect
github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
github.com/hashicorp/go-sockaddr v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/hashicorp/vault/api v1.9.0 // indirect
github.com/jellydator/ttlcache/v2 v2.11.1 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/ryanuber/go-glob v1.0.0 // indirect
github.com/skeema/knownhosts v1.1.0 // indirect
github.com/theupdateframework/go-tuf v0.5.2 // indirect
github.com/zeebo/errs v1.3.0 // indirect
Expand Down Expand Up @@ -193,12 +194,12 @@ require (
go.uber.org/multierr v1.8.0 // indirect
golang.org/x/crypto v0.6.0
golang.org/x/mod v0.8.0 // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/net v0.8.0
golang.org/x/sync v0.1.0
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/time v0.3.0
golang.org/x/tools v0.6.0 // indirect
google.golang.org/api v0.110.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

89 changes: 68 additions & 21 deletions pkg/apis/pipeline/v1/matrix_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@ package v1
import (
"context"
"fmt"
"sort"

"github.com/tektoncd/pipeline/pkg/apis/config"
"golang.org/x/exp/maps"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/utils/strings/slices"
"knative.dev/pkg/apis"
Expand All @@ -37,6 +39,11 @@ type Matrix struct {
// +optional
// +listType=atomic
Include []MatrixInclude `json:"include,omitempty"`

// combinations is a list of combinations created based on a given Matrix configurations including
// Matrix.Params and Matrix.Include.Params
// Combinations is skipped while encoding JSON since they are calculated based on the specified params
Combinations Combinations `json:"-"`
}

// MatrixInclude allows passing in a specific combinations of Parameters into the Matrix.
Expand All @@ -51,49 +58,89 @@ type MatrixInclude struct {
Params Params `json:"params,omitempty"`
}

// FanOut produces combinations of Parameters of type String from a slice of Parameters of type Array.
func (m *Matrix) FanOut() []Params {
var combinations []Params
for _, parameter := range m.Params {
combinations = fanOut(parameter, combinations)
// Combination holds a single combination from a Matrix with key as param.Name and value as param.Value
type Combination map[string]string

// Combinations holds a list of combination for a give Matrix
type Combinations []Combination

// ToParams transforms Combinations from a slice of map[string]string to a slice of Params
// such that, these combinations can be directly consumed in creating taskRun/run object
func (cs Combinations) ToParams() []Params {
listOfParams := make([]Params, len(cs))
for i := range cs {
var params Params
combination := cs[i]
order, _ := combination.sortCombination()
for _, key := range order {
params = append(params, Param{
Name: key,
Value: ParamValue{Type: ParamTypeString, StringVal: combination[key]},
})
}
listOfParams[i] = params
}
return combinations
return listOfParams
}

// fanOut generates a new combination based on a given Parameter in the Matrix.
func fanOut(param Param, combinations []Params) []Params {
if len(combinations) == 0 {
func (cs Combinations) fanOut(param Param) Combinations {
if len(cs) == 0 {
return initializeCombinations(param)
}
return distribute(param, combinations)
return cs.distribute(param)
}

// distribute generates a new combination of Parameters by adding a new Parameter to an existing list of Combinations.
func distribute(param Param, combinations []Params) []Params {
var expandedCombinations []Params
func (cs Combinations) distribute(param Param) Combinations {
var expandedCombinations Combinations
for _, value := range param.Value.ArrayVal {
for _, combination := range combinations {
expandedCombinations = append(expandedCombinations, createCombination(param.Name, value, combination))
for _, combination := range cs {
newCombination := make(Combination)
maps.Copy(newCombination, combination)
newCombination[param.Name] = value
_, orderedCombination := newCombination.sortCombination()
expandedCombinations = append(expandedCombinations, orderedCombination)
}
}
return expandedCombinations
}

// initializeCombinations generates a new combination based on the first Parameter in the Matrix.
func initializeCombinations(param Param) []Params {
var combinations []Params
func initializeCombinations(param Param) Combinations {
var combinations Combinations
for _, value := range param.Value.ArrayVal {
combinations = append(combinations, createCombination(param.Name, value, []Param{}))
combinations = append(combinations, Combination{param.Name: value})
}
return combinations
}

func createCombination(name string, value string, combination Params) Params {
combination = append(combination, Param{
Name: name,
Value: ParamValue{Type: ParamTypeString, StringVal: value},
// sortCombination sorts the given Combination based on the param names to produce a deterministic ordering
func (c Combination) sortCombination() ([]string, Combination) {
sortedCombination := make(Combination, len(c))
order := make([]string, 0, len(c))
for key := range c {
order = append(order, key)
}
sort.Slice(order, func(i, j int) bool {
if order[i] > order[j] {
return false
}
return true
})
return combination
for _, key := range order {
sortedCombination[key] = c[key]
}
return order, sortedCombination
}

// FanOut produces combinations of Parameters of type String from a slice of Parameters of type Array.
func (m *Matrix) FanOut() {
var combinations Combinations
for _, parameter := range m.Params {
combinations = combinations.fanOut(parameter)
}
m.Combinations = combinations
}

// CountCombinations returns the count of combinations of Parameters generated from the Matrix in PipelineTask.
Expand Down
Loading

0 comments on commit 70949c5

Please sign in to comment.