From 77c33cd246ecbc54ac5dd7df1765ae3c38ea9546 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Fri, 22 Nov 2024 15:45:54 +0530 Subject: [PATCH 01/17] [KEP-0009] feat: add expression based assertions This PR adds CEL-expression based assertions to `TestAsserts`. See https://github.com/kudobuilder/kuttl/blob/main/keps/0009-expression-based-assertions.md for more details. Signed-off-by: Kumar Mallikarjuna --- go.mod | 6 + go.sum | 8 +- pkg/apis/testharness/v1beta1/test_types.go | 17 +++ .../v1beta1/zz_generated.deepcopy.go | 48 ++++++++ pkg/test/case.go | 17 +-- pkg/test/step.go | 9 ++ pkg/test/utils/kubernetes.go | 103 ++++++++++++++++++ 7 files changed, 192 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 9d4927ce..73ba8f23 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/docker/docker v27.3.1+incompatible github.com/dustin/go-humanize v1.0.1 github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 + github.com/google/cel-go v0.20.1 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/spf13/cobra v1.8.1 @@ -30,6 +31,7 @@ require ( github.com/BurntSushi/toml v1.4.0 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect github.com/alessio/shellescape v1.4.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect @@ -70,12 +72,14 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect @@ -85,6 +89,8 @@ require ( golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 805bc0a9..bcec56a7 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -70,6 +72,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -165,12 +169,15 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -255,7 +262,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index 28d3c518..28165236 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -152,6 +152,10 @@ type TestAssert struct { Collectors []*TestCollector `json:"collectors,omitempty"` // Commands is a set of commands to be run as assertions for the current step Commands []TestAssertCommand `json:"commands,omitempty"` + + ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"` + + AssertExpressions AnyAllExpressions `json:"assertExpressions,omitempty"` } // TestAssertCommand an assertion based on the result of the execution of a command @@ -222,6 +226,19 @@ type TestCollector struct { Cmd string `json:"command,omitempty"` } +type TestResourceRef struct { + ApiVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Id string `json:"id,omitempty"` +} + +type AnyAllExpressions struct { + Any []string `json:"any,omitempty"` + All []string `json:"all,omitempty"` +} + // DefaultKINDContext defines the default kind context to use. const DefaultKINDContext = "kind" diff --git a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go index 4698a0c6..ca953a91 100644 --- a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go @@ -24,6 +24,32 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AnyAllExpressions) DeepCopyInto(out *AnyAllExpressions) { + *out = *in + if in.Any != nil { + in, out := &in.Any, &out.Any + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.All != nil { + in, out := &in.All, &out.All + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AnyAllExpressions. +func (in *AnyAllExpressions) DeepCopy() *AnyAllExpressions { + if in == nil { + return nil + } + out := new(AnyAllExpressions) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Command) DeepCopyInto(out *Command) { *out = *in @@ -95,6 +121,12 @@ func (in *TestAssert) DeepCopyInto(out *TestAssert) { *out = make([]TestAssertCommand, len(*in)) copy(*out, *in) } + if in.ResourceRefs != nil { + in, out := &in.ResourceRefs, &out.ResourceRefs + *out = make([]TestResourceRef, len(*in)) + copy(*out, *in) + } + in.AssertExpressions.DeepCopyInto(&out.AssertExpressions) return } @@ -179,6 +211,22 @@ func (in *TestFile) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestResourceRef) DeepCopyInto(out *TestResourceRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestResourceRef. +func (in *TestResourceRef) DeepCopy() *TestResourceRef { + if in == nil { + return nil + } + out := new(TestResourceRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestStep) DeepCopyInto(out *TestStep) { *out = *in diff --git a/pkg/test/case.go b/pkg/test/case.go index bf424299..242cfbf7 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -339,7 +339,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { continue } - cl, err = newClient(testStep.Kubeconfig, testStep.Context)(false) + cl, err = testutils.NewClient(testStep.Kubeconfig, testStep.Context)(false) if err != nil { setupReport.Failure = report.NewFailure(err.Error(), nil) ts.AddTestcase(setupReport) @@ -364,7 +364,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { tc := report.NewCase("step " + testStep.String()) testStep.Client = t.Client if testStep.Kubeconfig != "" { - testStep.Client = newClient(testStep.Kubeconfig, testStep.Context) + testStep.Client = testutils.NewClient(testStep.Kubeconfig, testStep.Context) } testStep.DiscoveryClient = t.DiscoveryClient if testStep.Kubeconfig != "" { @@ -532,19 +532,6 @@ func (t *Case) LoadTestSteps() error { return nil } -func newClient(kubeconfig, context string) func(bool) (client.Client, error) { - return func(bool) (client.Client, error) { - config, err := k8s.BuildConfigWithContext(kubeconfig, context) - if err != nil { - return nil, err - } - - return testutils.NewRetryClient(config, client.Options{ - Scheme: testutils.Scheme(), - }) - } -} - func newDiscoveryClient(kubeconfig, context string) func() (discovery.DiscoveryInterface, error) { return func() (discovery.DiscoveryInterface, error) { config, err := k8s.BuildConfigWithContext(kubeconfig, context) diff --git a/pkg/test/step.go b/pkg/test/step.go index 2e722f3a..1a7e89b6 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -412,6 +412,14 @@ func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, comman return testErrors } +func (s *Step) CheckAssertExpressions( + ctx context.Context, + resourceRefs []harness.TestResourceRef, + expressions harness.AnyAllExpressions, +) []error { + return testutils.RunAssertExpressions(ctx, s.Logger, resourceRefs, expressions, s.Kubeconfig) +} + // Check checks if the resources defined in Asserts and Errors are in the correct state. func (s *Step) Check(namespace string, timeout int) []error { testErrors := []error{} @@ -422,6 +430,7 @@ func (s *Step) Check(namespace string, timeout int) []error { if s.Assert != nil { testErrors = append(testErrors, s.CheckAssertCommands(context.TODO(), namespace, s.Assert.Commands, timeout)...) + testErrors = append(testErrors, s.CheckAssertExpressions(context.TODO(), s.Assert.ResourceRefs, s.Assert.AssertExpressions)...) } for _, expected := range s.Errors { diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 4c703691..583eacf1 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/google/cel-go/cel" "github.com/google/shlex" "github.com/pmezard/go-difflib/difflib" "github.com/spf13/pflag" @@ -57,6 +58,7 @@ import ( "github.com/kudobuilder/kuttl/pkg/apis" harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" "github.com/kudobuilder/kuttl/pkg/env" + "github.com/kudobuilder/kuttl/pkg/k8s" ) // ensure that we only add to the scheme once. @@ -1217,6 +1219,81 @@ func RunAssertCommands(ctx context.Context, logger Logger, namespace string, com return RunCommands(ctx, logger, namespace, convertAssertCommand(commands, timeout), workdir, timeout, kubeconfigOverride) } +// RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions +func RunAssertExpressions( + ctx context.Context, + logger Logger, + resourceRefs []harness.TestResourceRef, + expressions harness.AnyAllExpressions, + kubeconfigOverride string, +) []error { + errs := []error{} + + actualDir, err := os.Getwd() + if err != nil { + return []error{fmt.Errorf("failed to get current working director: %w", err)} + } + + kubeconfig := kubeconfigPath(actualDir, kubeconfigOverride) + cl, err := NewClient(kubeconfig, "")(false) + if err != nil { + return []error{fmt.Errorf("failed to construct client: %w", err)} + } + + variables := make(map[string]interface{}) + for _, resourceRef := range resourceRefs { + gvk := constructGVK(resourceRef.ApiVersion, resourceRef.Kind) + referencedResource := &unstructured.Unstructured{} + referencedResource.SetGroupVersionKind(gvk) + + if err := cl.Get( + ctx, + types.NamespacedName{Namespace: resourceRef.Namespace, Name: resourceRef.Name}, + referencedResource, + ); err != nil { + return []error{fmt.Errorf("failed to get referenced resource '%v': %w", gvk, err)} + } + + variables[resourceRef.Id] = referencedResource.Object + } + + env, err := cel.NewEnv() + if err != nil { + return []error{fmt.Errorf("failed to create environment: %w", err)} + } + + for k := range variables { + env, err = env.Extend(cel.Variable(k, cel.DynType)) + if err != nil { + return []error{fmt.Errorf("failed to add resource parameter '%v' to environment: %w", k, err)} + } + } + + for _, expr := range expressions.Any { + ast, issues := env.Compile(expr) + if issues != nil && issues.Err() != nil { + return []error{fmt.Errorf("type-check error: %s", issues.Err())} + } + + prg, err := env.Program(ast) + if err != nil { + return []error{fmt.Errorf("program constuction error: %w", err)} + } + + out, _, err := prg.Eval(variables) + if err != nil { + return []error{fmt.Errorf("failed to evaluate program: %w", err)} + } + + logger.Logf("expression '%v' evaluated to '%v'", expr, out.Value()) + if out.Value() != true { + errs = append(errs, fmt.Errorf("failed validation, expression '%v' evaluated to '%v'", expr, out.Value())) + } + } + + return errs +} + // RunCommands runs a set of commands, returning any errors. // If any (non-background) command fails, the following commands are skipped // commands running in the background are returned @@ -1321,3 +1398,29 @@ func Kubeconfig(cfg *rest.Config, w io.Writer) error { }, }, w) } + +func constructGVK(apiVersion, kind string) schema.GroupVersionKind { + apiVersionSplit := strings.Split(apiVersion, "/") + gvk := schema.GroupVersionKind{ + Version: apiVersionSplit[len(apiVersionSplit)-1], + Kind: kind, + } + if len(apiVersion) > 1 { + gvk.Group = apiVersionSplit[0] + } + + return gvk +} + +func NewClient(kubeconfig, context string) func(bool) (client.Client, error) { + return func(bool) (client.Client, error) { + config, err := k8s.BuildConfigWithContext(kubeconfig, context) + if err != nil { + return nil, err + } + + return NewRetryClient(config, client.Options{ + Scheme: Scheme(), + }) + } +} From fd5aae1975a8f7494c06638dc3d626082722c73c Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 27 Nov 2024 11:02:41 +0530 Subject: [PATCH 02/17] fix: prevent redundant client construction Signed-off-by: Kumar Mallikarjuna --- pkg/test/utils/kubernetes.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 583eacf1..05d25969 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1228,6 +1228,9 @@ func RunAssertExpressions( kubeconfigOverride string, ) []error { errs := []error{} + if len(expressions.Any) == 0 && len(expressions.All) == 0 { + return errs + } actualDir, err := os.Getwd() if err != nil { From 0e43475600c234fdaaeb14dbc5180f178fa5f171 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 27 Nov 2024 11:08:18 +0530 Subject: [PATCH 03/17] chore: rename Id->Ref and make linter happy Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/test_types.go | 4 ++-- pkg/test/utils/kubernetes.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index 28165236..55a4f3b9 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -227,11 +227,11 @@ type TestCollector struct { } type TestResourceRef struct { - ApiVersion string `json:"apiVersion,omitempty"` + APIVersion string `json:"apiVersion,omitempty"` Kind string `json:"kind,omitempty"` Namespace string `json:"namespace,omitempty"` Name string `json:"name,omitempty"` - Id string `json:"id,omitempty"` + Ref string `json:"ref,omitempty"` } type AnyAllExpressions struct { diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 05d25969..b2e34e06 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1245,7 +1245,7 @@ func RunAssertExpressions( variables := make(map[string]interface{}) for _, resourceRef := range resourceRefs { - gvk := constructGVK(resourceRef.ApiVersion, resourceRef.Kind) + gvk := constructGVK(resourceRef.APIVersion, resourceRef.Kind) referencedResource := &unstructured.Unstructured{} referencedResource.SetGroupVersionKind(gvk) @@ -1257,7 +1257,7 @@ func RunAssertExpressions( return []error{fmt.Errorf("failed to get referenced resource '%v': %w", gvk, err)} } - variables[resourceRef.Id] = referencedResource.Object + variables[resourceRef.Ref] = referencedResource.Object } env, err := cel.NewEnv() @@ -1280,7 +1280,7 @@ func RunAssertExpressions( prg, err := env.Program(ast) if err != nil { - return []error{fmt.Errorf("program constuction error: %w", err)} + return []error{fmt.Errorf("program construction error: %w", err)} } out, _, err := prg.Eval(variables) From dc2b416ee0c4afc20d174f9e5482dc812ad122ad Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 27 Nov 2024 11:57:49 +0530 Subject: [PATCH 04/17] refactor: add method for building resource ref Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 30 ++++++++++++++++++++++ pkg/test/utils/kubernetes.go | 21 +++------------ 2 files changed, 33 insertions(+), 18 deletions(-) create mode 100644 pkg/apis/testharness/v1beta1/expression.go diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go new file mode 100644 index 00000000..7185eb64 --- /dev/null +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -0,0 +1,30 @@ +package v1beta1 + +import ( + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +func (t *TestResourceRef) BuildResourceReference() (types.NamespacedName, *unstructured.Unstructured) { + apiVersionSplit := strings.Split(t.APIVersion, "/") + gvk := schema.GroupVersionKind{ + Version: apiVersionSplit[len(apiVersionSplit)-1], + Kind: t.Kind, + } + if len(t.APIVersion) > 1 { + gvk.Group = apiVersionSplit[0] + } + + referencedResource := &unstructured.Unstructured{} + referencedResource.SetGroupVersionKind(gvk) + + namespacedName := types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Name, + } + + return namespacedName, referencedResource +} diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index b2e34e06..8794a8e8 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1245,16 +1245,14 @@ func RunAssertExpressions( variables := make(map[string]interface{}) for _, resourceRef := range resourceRefs { - gvk := constructGVK(resourceRef.APIVersion, resourceRef.Kind) - referencedResource := &unstructured.Unstructured{} - referencedResource.SetGroupVersionKind(gvk) + namespacedName, referencedResource := resourceRef.BuildResourceReference() if err := cl.Get( ctx, - types.NamespacedName{Namespace: resourceRef.Namespace, Name: resourceRef.Name}, + namespacedName, referencedResource, ); err != nil { - return []error{fmt.Errorf("failed to get referenced resource '%v': %w", gvk, err)} + return []error{fmt.Errorf("failed to get referenced resource '%v': %w", namespacedName, err)} } variables[resourceRef.Ref] = referencedResource.Object @@ -1402,19 +1400,6 @@ func Kubeconfig(cfg *rest.Config, w io.Writer) error { }, w) } -func constructGVK(apiVersion, kind string) schema.GroupVersionKind { - apiVersionSplit := strings.Split(apiVersion, "/") - gvk := schema.GroupVersionKind{ - Version: apiVersionSplit[len(apiVersionSplit)-1], - Kind: kind, - } - if len(apiVersion) > 1 { - gvk.Group = apiVersionSplit[0] - } - - return gvk -} - func NewClient(kubeconfig, context string) func(bool) (client.Client, error) { return func(bool) (client.Client, error) { config, err := k8s.BuildConfigWithContext(kubeconfig, context) From 2e36db675bed07e70ab165b0abd13caca1e06d3e Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 27 Nov 2024 14:14:27 +0530 Subject: [PATCH 05/17] chore: add validation for resourceRefs Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 39 +++++++++++++++++++--- pkg/test/step.go | 11 ++++++ pkg/test/utils/kubernetes.go | 1 - 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go index 7185eb64..762f56ed 100644 --- a/pkg/apis/testharness/v1beta1/expression.go +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -1,6 +1,8 @@ package v1beta1 import ( + "errors" + "fmt" "strings" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -8,7 +10,8 @@ import ( "k8s.io/apimachinery/pkg/types" ) -func (t *TestResourceRef) BuildResourceReference() (types.NamespacedName, *unstructured.Unstructured) { +func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) { + referencedResource = &unstructured.Unstructured{} apiVersionSplit := strings.Split(t.APIVersion, "/") gvk := schema.GroupVersionKind{ Version: apiVersionSplit[len(apiVersionSplit)-1], @@ -17,14 +20,40 @@ func (t *TestResourceRef) BuildResourceReference() (types.NamespacedName, *unstr if len(t.APIVersion) > 1 { gvk.Group = apiVersionSplit[0] } - - referencedResource := &unstructured.Unstructured{} referencedResource.SetGroupVersionKind(gvk) - namespacedName := types.NamespacedName{ + namespacedName = types.NamespacedName{ Namespace: t.Namespace, Name: t.Name, } - return namespacedName, referencedResource + return +} + +func (t *TestResourceRef) ValidateResourceReference() error { + apiVersionSplit := strings.Split(t.APIVersion, "/") + if t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2) { + return fmt.Errorf("apiVersion '%v' not of the format (/)", t.APIVersion) + } else if t.Kind == "" { + return errors.New("kind not specified") + } else if t.Namespace == "" { + return errors.New("namespace not specified") + } else if t.Name == "" { + return errors.New("name not specified") + } else if t.Ref == "" { + return errors.New("ref not specified") + } + + return nil +} + +func (t *TestResourceRef) String() string { + return fmt.Sprintf( + "apiVersion=%v, kind=%v, namespace=%v, name=%v, ref=%v", + t.APIVersion, + t.Kind, + t.Namespace, + t.Name, + t.Ref, + ) } diff --git a/pkg/test/step.go b/pkg/test/step.go index 1a7e89b6..af96f4fe 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -542,6 +542,17 @@ func (s *Step) LoadYAML(file string) error { } else { return fmt.Errorf("failed to load TestAssert object from %s: it contains an object of type %T", file, obj) } + + var errs []error + for _, resourceRef := range s.Assert.ResourceRefs { + if err := resourceRef.ValidateResourceReference(); err != nil { + errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err)) + } + } + + if len(errs) > 0 { + return fmt.Errorf("failed to load TestAssert object from %s: %w", file, errors.Join(errs...)) + } } else { asserts = append(asserts, obj) } diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 8794a8e8..da5214b7 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1246,7 +1246,6 @@ func RunAssertExpressions( variables := make(map[string]interface{}) for _, resourceRef := range resourceRefs { namespacedName, referencedResource := resourceRef.BuildResourceReference() - if err := cl.Get( ctx, namespacedName, From 52b4835c2e06759ba4be9ab0c6e683a9bd39c80c Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 27 Nov 2024 14:31:00 +0530 Subject: [PATCH 06/17] chore: make assertion syntax consistent with the KEP Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/test_types.go | 10 ++++--- .../v1beta1/zz_generated.deepcopy.go | 29 +++++++++---------- pkg/test/step.go | 7 +++-- pkg/test/utils/kubernetes.go | 9 +++--- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index 55a4f3b9..e152e99b 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -9,6 +9,8 @@ import ( const KubeconfigLoadingEager = "Eager" const KubeconfigLoadingLazy = "Lazy" +type CELExpression string + // Create embedded struct to implement custom DeepCopyInto method type RestConfig struct { RC *rest.Config @@ -155,7 +157,8 @@ type TestAssert struct { ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"` - AssertExpressions AnyAllExpressions `json:"assertExpressions,omitempty"` + AssertAny []Assertion `json:"assertAny,omitempty"` + AssertAll []Assertion `json:"assertAll,omitempty"` } // TestAssertCommand an assertion based on the result of the execution of a command @@ -234,9 +237,8 @@ type TestResourceRef struct { Ref string `json:"ref,omitempty"` } -type AnyAllExpressions struct { - Any []string `json:"any,omitempty"` - All []string `json:"all,omitempty"` +type Assertion struct { + CELExpression CELExpression `json:"celExpr,omitempty"` } // DefaultKINDContext defines the default kind context to use. diff --git a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go index ca953a91..0966036f 100644 --- a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go @@ -25,27 +25,17 @@ import ( ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *AnyAllExpressions) DeepCopyInto(out *AnyAllExpressions) { +func (in *Assertion) DeepCopyInto(out *Assertion) { *out = *in - if in.Any != nil { - in, out := &in.Any, &out.Any - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.All != nil { - in, out := &in.All, &out.All - *out = make([]string, len(*in)) - copy(*out, *in) - } return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AnyAllExpressions. -func (in *AnyAllExpressions) DeepCopy() *AnyAllExpressions { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Assertion. +func (in *Assertion) DeepCopy() *Assertion { if in == nil { return nil } - out := new(AnyAllExpressions) + out := new(Assertion) in.DeepCopyInto(out) return out } @@ -126,7 +116,16 @@ func (in *TestAssert) DeepCopyInto(out *TestAssert) { *out = make([]TestResourceRef, len(*in)) copy(*out, *in) } - in.AssertExpressions.DeepCopyInto(&out.AssertExpressions) + if in.AssertAny != nil { + in, out := &in.AssertAny, &out.AssertAny + *out = make([]Assertion, len(*in)) + copy(*out, *in) + } + if in.AssertAll != nil { + in, out := &in.AssertAll, &out.AssertAll + *out = make([]Assertion, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/test/step.go b/pkg/test/step.go index af96f4fe..a4682c62 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -415,9 +415,10 @@ func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, comman func (s *Step) CheckAssertExpressions( ctx context.Context, resourceRefs []harness.TestResourceRef, - expressions harness.AnyAllExpressions, + assertAny, + assertAll []harness.Assertion, ) []error { - return testutils.RunAssertExpressions(ctx, s.Logger, resourceRefs, expressions, s.Kubeconfig) + return testutils.RunAssertExpressions(ctx, s.Logger, resourceRefs, assertAny, assertAll, s.Kubeconfig) } // Check checks if the resources defined in Asserts and Errors are in the correct state. @@ -430,7 +431,7 @@ func (s *Step) Check(namespace string, timeout int) []error { if s.Assert != nil { testErrors = append(testErrors, s.CheckAssertCommands(context.TODO(), namespace, s.Assert.Commands, timeout)...) - testErrors = append(testErrors, s.CheckAssertExpressions(context.TODO(), s.Assert.ResourceRefs, s.Assert.AssertExpressions)...) + testErrors = append(testErrors, s.CheckAssertExpressions(context.TODO(), s.Assert.ResourceRefs, s.Assert.AssertAny, s.Assert.AssertAll)...) } for _, expected := range s.Errors { diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index da5214b7..ba0104e5 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1224,11 +1224,12 @@ func RunAssertExpressions( ctx context.Context, logger Logger, resourceRefs []harness.TestResourceRef, - expressions harness.AnyAllExpressions, + assertAny, + assertAll []harness.Assertion, kubeconfigOverride string, ) []error { errs := []error{} - if len(expressions.Any) == 0 && len(expressions.All) == 0 { + if len(assertAny) == 0 && len(assertAll) == 0 { return errs } @@ -1269,8 +1270,8 @@ func RunAssertExpressions( } } - for _, expr := range expressions.Any { - ast, issues := env.Compile(expr) + for _, expr := range assertAny { + ast, issues := env.Compile(string(expr.CELExpression)) if issues != nil && issues.Err() != nil { return []error{fmt.Errorf("type-check error: %s", issues.Err())} } From 5dcbef929615c3bddec4bbd00226e2c6169aeec0 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 27 Nov 2024 14:33:12 +0530 Subject: [PATCH 07/17] refactor: rename Validate method for `TestResourceRef` Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 2 +- pkg/test/step.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go index 762f56ed..dd2bc3af 100644 --- a/pkg/apis/testharness/v1beta1/expression.go +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -30,7 +30,7 @@ func (t *TestResourceRef) BuildResourceReference() (namespacedName types.Namespa return } -func (t *TestResourceRef) ValidateResourceReference() error { +func (t *TestResourceRef) Validate() error { apiVersionSplit := strings.Split(t.APIVersion, "/") if t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2) { return fmt.Errorf("apiVersion '%v' not of the format (/)", t.APIVersion) diff --git a/pkg/test/step.go b/pkg/test/step.go index a4682c62..cdcc7dbb 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -546,7 +546,7 @@ func (s *Step) LoadYAML(file string) error { var errs []error for _, resourceRef := range s.Assert.ResourceRefs { - if err := resourceRef.ValidateResourceReference(); err != nil { + if err := resourceRef.Validate(); err != nil { errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err)) } } From a7e52870acb1740f4fb9079301b30321a210c316 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Tue, 3 Dec 2024 13:16:00 +0530 Subject: [PATCH 08/17] chore: pre-build environment and program for expressions Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 15 +++++ pkg/apis/testharness/v1beta1/test_types.go | 8 +-- .../v1beta1/zz_generated.deepcopy.go | 20 ++++-- pkg/test/step.go | 35 ++++++++++- pkg/test/utils/expression.go | 25 ++++++++ pkg/test/utils/kubernetes.go | 63 +++++++++---------- pkg/test/utils/kubernetes_test.go | 2 +- 7 files changed, 122 insertions(+), 46 deletions(-) create mode 100644 pkg/test/utils/expression.go diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go index dd2bc3af..6c265099 100644 --- a/pkg/apis/testharness/v1beta1/expression.go +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/google/cel-go/cel" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -57,3 +58,17 @@ func (t *TestResourceRef) String() string { t.Ref, ) } + +func (t *Assertion) BuildProgram(env *cel.Env) (cel.Program, error) { + ast, issues := env.Compile(t.CELExpression) + if issues != nil && issues.Err() != nil { + return nil, fmt.Errorf("type-check error: %s", issues.Err()) + } + + prg, err := env.Program(ast) + if err != nil { + return nil, fmt.Errorf("program construction error: %w", err) + } + + return prg, nil +} diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index e152e99b..6e2f1cd8 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -9,8 +9,6 @@ import ( const KubeconfigLoadingEager = "Eager" const KubeconfigLoadingLazy = "Lazy" -type CELExpression string - // Create embedded struct to implement custom DeepCopyInto method type RestConfig struct { RC *rest.Config @@ -157,8 +155,8 @@ type TestAssert struct { ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"` - AssertAny []Assertion `json:"assertAny,omitempty"` - AssertAll []Assertion `json:"assertAll,omitempty"` + AssertAny []*Assertion `json:"assertAny,omitempty"` + AssertAll []*Assertion `json:"assertAll,omitempty"` } // TestAssertCommand an assertion based on the result of the execution of a command @@ -238,7 +236,7 @@ type TestResourceRef struct { } type Assertion struct { - CELExpression CELExpression `json:"celExpr,omitempty"` + CELExpression string `json:"celExpr,omitempty"` } // DefaultKINDContext defines the default kind context to use. diff --git a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go index 0966036f..303263f4 100644 --- a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go @@ -118,13 +118,25 @@ func (in *TestAssert) DeepCopyInto(out *TestAssert) { } if in.AssertAny != nil { in, out := &in.AssertAny, &out.AssertAny - *out = make([]Assertion, len(*in)) - copy(*out, *in) + *out = make([]*Assertion, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Assertion) + **out = **in + } + } } if in.AssertAll != nil { in, out := &in.AssertAll, &out.AssertAll - *out = make([]Assertion, len(*in)) - copy(*out, *in) + *out = make([]*Assertion, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Assertion) + **out = **in + } + } } return } diff --git a/pkg/test/step.go b/pkg/test/step.go index cdcc7dbb..9557ef83 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/google/cel-go/cel" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,6 +46,8 @@ type Step struct { Step *harness.TestStep Assert *harness.TestAssert + Programs map[string]cel.Program + Asserts []client.Object Apply []client.Object Errors []client.Object @@ -416,9 +419,14 @@ func (s *Step) CheckAssertExpressions( ctx context.Context, resourceRefs []harness.TestResourceRef, assertAny, - assertAll []harness.Assertion, + assertAll []*harness.Assertion, ) []error { - return testutils.RunAssertExpressions(ctx, s.Logger, resourceRefs, assertAny, assertAll, s.Kubeconfig) + client, err := s.Client(false) + if err != nil { + return []error{err} + } + + return testutils.RunAssertExpressions(ctx, s.Logger, client, s.Programs, resourceRefs, assertAny, assertAll) } // Check checks if the resources defined in Asserts and Errors are in the correct state. @@ -525,6 +533,8 @@ func (s *Step) String() string { // if seen, mark a test immediately failed. // - All other YAML files are considered resources to create. func (s *Step) LoadYAML(file string) error { + s.Programs = make(map[string]cel.Program) + skipFile, objects, err := s.loadOrSkipFile(file) if skipFile || err != nil { return err @@ -554,6 +564,27 @@ func (s *Step) LoadYAML(file string) error { if len(errs) > 0 { return fmt.Errorf("failed to load TestAssert object from %s: %w", file, errors.Join(errs...)) } + + var assertions []*harness.Assertion + assertions = append(assertions, s.Assert.AssertAny...) + assertions = append(assertions, s.Assert.AssertAll...) + + env, err := testutils.BuildEnv(s.Assert.ResourceRefs) + if err != nil { + return fmt.Errorf("failed to load TestAssert object from %s: %w", file, err) + } + + for _, assertion := range assertions { + if prg, err := assertion.BuildProgram(env); err != nil { + errs = append(errs, err) + } else { + s.Programs[assertion.CELExpression] = prg + } + } + + if len(errs) > 0 { + return fmt.Errorf("failed to load TestAssert object from %s: %w", file, errors.Join(errs...)) + } } else { asserts = append(asserts, obj) } diff --git a/pkg/test/utils/expression.go b/pkg/test/utils/expression.go new file mode 100644 index 00000000..75b812f5 --- /dev/null +++ b/pkg/test/utils/expression.go @@ -0,0 +1,25 @@ +package utils + +import ( + "fmt" + + "github.com/google/cel-go/cel" + + "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" +) + +func BuildEnv(resourceRefs []v1beta1.TestResourceRef) (*cel.Env, error) { + env, err := cel.NewEnv() + if err != nil { + return nil, fmt.Errorf("failed to create environment: %w", err) + } + + for _, resourceRef := range resourceRefs { + env, err = env.Extend(cel.Variable(resourceRef.Ref, cel.DynType)) + if err != nil { + return nil, fmt.Errorf("failed to add resource parameter '%v' to environment: %w", resourceRef.Ref, err) + } + } + + return env, nil +} diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index ba0104e5..23d59da4 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1114,7 +1114,7 @@ func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd kuttlENV := make(map[string]string) kuttlENV["NAMESPACE"] = namespace - kuttlENV["KUBECONFIG"] = kubeconfigPath(actualDir, kubeconfigOverride) + kuttlENV["KUBECONFIG"] = KubeconfigPath(actualDir, kubeconfigOverride) kuttlENV["PATH"] = fmt.Sprintf("%s/bin/:%s", actualDir, os.Getenv("PATH")) // by default testsuite timeout is the command timeout @@ -1183,7 +1183,7 @@ func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd return nil, nil } -func kubeconfigPath(actualDir, override string) string { +func KubeconfigPath(actualDir, override string) string { if override != "" { if filepath.IsAbs(override) { return override @@ -1223,27 +1223,17 @@ func RunAssertCommands(ctx context.Context, logger Logger, namespace string, com func RunAssertExpressions( ctx context.Context, logger Logger, + cl client.Client, + programs map[string]cel.Program, resourceRefs []harness.TestResourceRef, assertAny, - assertAll []harness.Assertion, - kubeconfigOverride string, + assertAll []*harness.Assertion, ) []error { errs := []error{} if len(assertAny) == 0 && len(assertAll) == 0 { return errs } - actualDir, err := os.Getwd() - if err != nil { - return []error{fmt.Errorf("failed to get current working director: %w", err)} - } - - kubeconfig := kubeconfigPath(actualDir, kubeconfigOverride) - cl, err := NewClient(kubeconfig, "")(false) - if err != nil { - return []error{fmt.Errorf("failed to construct client: %w", err)} - } - variables := make(map[string]interface{}) for _, resourceRef := range resourceRefs { namespacedName, referencedResource := resourceRef.BuildResourceReference() @@ -1258,40 +1248,45 @@ func RunAssertExpressions( variables[resourceRef.Ref] = referencedResource.Object } - env, err := cel.NewEnv() - if err != nil { - return []error{fmt.Errorf("failed to create environment: %w", err)} - } - - for k := range variables { - env, err = env.Extend(cel.Variable(k, cel.DynType)) + var anyExpressionsEvaluation, allExpressionsEvaluation []error + for _, expr := range assertAny { + prg, ok := programs[expr.CELExpression] + if !ok { + return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} + } + out, _, err := prg.Eval(variables) if err != nil { - return []error{fmt.Errorf("failed to add resource parameter '%v' to environment: %w", k, err)} + return []error{fmt.Errorf("failed to evaluate program: %w", err)} } - } - for _, expr := range assertAny { - ast, issues := env.Compile(string(expr.CELExpression)) - if issues != nil && issues.Err() != nil { - return []error{fmt.Errorf("type-check error: %s", issues.Err())} + if out.Value() != true { + anyExpressionsEvaluation = append(anyExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) } + } - prg, err := env.Program(ast) - if err != nil { - return []error{fmt.Errorf("program construction error: %w", err)} + for _, expr := range assertAll { + prg, ok := programs[expr.CELExpression] + if !ok { + return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} } - out, _, err := prg.Eval(variables) if err != nil { return []error{fmt.Errorf("failed to evaluate program: %w", err)} } - logger.Logf("expression '%v' evaluated to '%v'", expr, out.Value()) if out.Value() != true { - errs = append(errs, fmt.Errorf("failed validation, expression '%v' evaluated to '%v'", expr, out.Value())) + allExpressionsEvaluation = append(allExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) } } + if len(assertAny) != 0 && len(anyExpressionsEvaluation) == len(assertAny) { + errs = append(errs, fmt.Errorf("no expression evaluated to true: %w", errors.Join(anyExpressionsEvaluation...))) + } + + if len(allExpressionsEvaluation) != len(assertAll) { + errs = append(errs, fmt.Errorf("not all expressions evaluated to true: %w", errors.Join(allExpressionsEvaluation...))) + } + return errs } diff --git a/pkg/test/utils/kubernetes_test.go b/pkg/test/utils/kubernetes_test.go index 1adcecf3..a464d4e4 100644 --- a/pkg/test/utils/kubernetes_test.go +++ b/pkg/test/utils/kubernetes_test.go @@ -134,7 +134,7 @@ func TestKubeconfigPath(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - result := kubeconfigPath(tt.path, tt.override) + result := KubeconfigPath(tt.path, tt.override) assert.Equal(t, tt.expected, result) }) } From a85752a5e81f59b6a392cb70a861f2df5a7c94ff Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Tue, 3 Dec 2024 13:25:22 +0530 Subject: [PATCH 09/17] chore: make linter happy and initialize Programs only if assertions are present Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 11 ++++++----- pkg/test/step.go | 8 +++++--- pkg/test/utils/kubernetes.go | 1 - 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go index 6c265099..0739de2f 100644 --- a/pkg/apis/testharness/v1beta1/expression.go +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -33,15 +33,16 @@ func (t *TestResourceRef) BuildResourceReference() (namespacedName types.Namespa func (t *TestResourceRef) Validate() error { apiVersionSplit := strings.Split(t.APIVersion, "/") - if t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2) { + switch { + case t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2): return fmt.Errorf("apiVersion '%v' not of the format (/)", t.APIVersion) - } else if t.Kind == "" { + case t.Kind == "": return errors.New("kind not specified") - } else if t.Namespace == "" { + case t.Namespace == "": return errors.New("namespace not specified") - } else if t.Name == "" { + case t.Name == "": return errors.New("name not specified") - } else if t.Ref == "" { + case t.Ref == "": return errors.New("ref not specified") } diff --git a/pkg/test/step.go b/pkg/test/step.go index 9557ef83..5043f7bd 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -426,7 +426,7 @@ func (s *Step) CheckAssertExpressions( return []error{err} } - return testutils.RunAssertExpressions(ctx, s.Logger, client, s.Programs, resourceRefs, assertAny, assertAll) + return testutils.RunAssertExpressions(ctx, client, s.Programs, resourceRefs, assertAny, assertAll) } // Check checks if the resources defined in Asserts and Errors are in the correct state. @@ -533,8 +533,6 @@ func (s *Step) String() string { // if seen, mark a test immediately failed. // - All other YAML files are considered resources to create. func (s *Step) LoadYAML(file string) error { - s.Programs = make(map[string]cel.Program) - skipFile, objects, err := s.loadOrSkipFile(file) if skipFile || err != nil { return err @@ -574,6 +572,10 @@ func (s *Step) LoadYAML(file string) error { return fmt.Errorf("failed to load TestAssert object from %s: %w", file, err) } + if len(assertions) > 0 { + s.Programs = make(map[string]cel.Program) + } + for _, assertion := range assertions { if prg, err := assertion.BuildProgram(env); err != nil { errs = append(errs, err) diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 23d59da4..4f2d0a24 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -1222,7 +1222,6 @@ func RunAssertCommands(ctx context.Context, logger Logger, namespace string, com // RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions func RunAssertExpressions( ctx context.Context, - logger Logger, cl client.Client, programs map[string]cel.Program, resourceRefs []harness.TestResourceRef, From 212b99313672a88cda164b41380129c077635761 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Tue, 3 Dec 2024 22:36:43 +0530 Subject: [PATCH 10/17] chore: incorporate review comments Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 19 +--------- .../expression.go => expressions/cel.go} | 16 +++++++- pkg/test/case.go | 17 ++++++++- pkg/test/step.go | 21 ++++++++-- pkg/test/utils/kubernetes.go | 38 ++----------------- pkg/test/utils/kubernetes_test.go | 2 +- 6 files changed, 54 insertions(+), 59 deletions(-) rename pkg/{test/utils/expression.go => expressions/cel.go} (60%) diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go index 0739de2f..de210a2d 100644 --- a/pkg/apis/testharness/v1beta1/expression.go +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/google/cel-go/cel" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -18,7 +17,7 @@ func (t *TestResourceRef) BuildResourceReference() (namespacedName types.Namespa Version: apiVersionSplit[len(apiVersionSplit)-1], Kind: t.Kind, } - if len(t.APIVersion) > 1 { + if len(apiVersionSplit) > 1 { gvk.Group = apiVersionSplit[0] } referencedResource.SetGroupVersionKind(gvk) @@ -38,8 +37,6 @@ func (t *TestResourceRef) Validate() error { return fmt.Errorf("apiVersion '%v' not of the format (/)", t.APIVersion) case t.Kind == "": return errors.New("kind not specified") - case t.Namespace == "": - return errors.New("namespace not specified") case t.Name == "": return errors.New("name not specified") case t.Ref == "": @@ -59,17 +56,3 @@ func (t *TestResourceRef) String() string { t.Ref, ) } - -func (t *Assertion) BuildProgram(env *cel.Env) (cel.Program, error) { - ast, issues := env.Compile(t.CELExpression) - if issues != nil && issues.Err() != nil { - return nil, fmt.Errorf("type-check error: %s", issues.Err()) - } - - prg, err := env.Program(ast) - if err != nil { - return nil, fmt.Errorf("program construction error: %w", err) - } - - return prg, nil -} diff --git a/pkg/test/utils/expression.go b/pkg/expressions/cel.go similarity index 60% rename from pkg/test/utils/expression.go rename to pkg/expressions/cel.go index 75b812f5..cd52fe7d 100644 --- a/pkg/test/utils/expression.go +++ b/pkg/expressions/cel.go @@ -1,4 +1,4 @@ -package utils +package expressions import ( "fmt" @@ -8,6 +8,20 @@ import ( "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" ) +func BuildProgram(expr string, env *cel.Env) (cel.Program, error) { + ast, issues := env.Compile(expr) + if issues != nil && issues.Err() != nil { + return nil, fmt.Errorf("type-check error: %s", issues.Err()) + } + + prg, err := env.Program(ast) + if err != nil { + return nil, fmt.Errorf("program construction error: %w", err) + } + + return prg, nil +} + func BuildEnv(resourceRefs []v1beta1.TestResourceRef) (*cel.Env, error) { env, err := cel.NewEnv() if err != nil { diff --git a/pkg/test/case.go b/pkg/test/case.go index 242cfbf7..bf424299 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -339,7 +339,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { continue } - cl, err = testutils.NewClient(testStep.Kubeconfig, testStep.Context)(false) + cl, err = newClient(testStep.Kubeconfig, testStep.Context)(false) if err != nil { setupReport.Failure = report.NewFailure(err.Error(), nil) ts.AddTestcase(setupReport) @@ -364,7 +364,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) { tc := report.NewCase("step " + testStep.String()) testStep.Client = t.Client if testStep.Kubeconfig != "" { - testStep.Client = testutils.NewClient(testStep.Kubeconfig, testStep.Context) + testStep.Client = newClient(testStep.Kubeconfig, testStep.Context) } testStep.DiscoveryClient = t.DiscoveryClient if testStep.Kubeconfig != "" { @@ -532,6 +532,19 @@ func (t *Case) LoadTestSteps() error { return nil } +func newClient(kubeconfig, context string) func(bool) (client.Client, error) { + return func(bool) (client.Client, error) { + config, err := k8s.BuildConfigWithContext(kubeconfig, context) + if err != nil { + return nil, err + } + + return testutils.NewRetryClient(config, client.Options{ + Scheme: testutils.Scheme(), + }) + } +} + func newDiscoveryClient(kubeconfig, context string) func() (discovery.DiscoveryInterface, error) { return func() (discovery.DiscoveryInterface, error) { config, err := k8s.BuildConfigWithContext(kubeconfig, context) diff --git a/pkg/test/step.go b/pkg/test/step.go index 5043f7bd..d8f2693a 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -24,6 +24,7 @@ import ( harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" "github.com/kudobuilder/kuttl/pkg/env" + "github.com/kudobuilder/kuttl/pkg/expressions" kfile "github.com/kudobuilder/kuttl/pkg/file" "github.com/kudobuilder/kuttl/pkg/http" testutils "github.com/kudobuilder/kuttl/pkg/test/utils" @@ -426,7 +427,21 @@ func (s *Step) CheckAssertExpressions( return []error{err} } - return testutils.RunAssertExpressions(ctx, client, s.Programs, resourceRefs, assertAny, assertAll) + variables := make(map[string]interface{}) + for _, resourceRef := range resourceRefs { + namespacedName, referencedResource := resourceRef.BuildResourceReference() + if err := client.Get( + ctx, + namespacedName, + referencedResource, + ); err != nil { + return []error{fmt.Errorf("failed to get referenced resource '%v': %w", namespacedName, err)} + } + + variables[resourceRef.Ref] = referencedResource.Object + } + + return testutils.RunAssertExpressions(s.Programs, variables, assertAny, assertAll) } // Check checks if the resources defined in Asserts and Errors are in the correct state. @@ -567,7 +582,7 @@ func (s *Step) LoadYAML(file string) error { assertions = append(assertions, s.Assert.AssertAny...) assertions = append(assertions, s.Assert.AssertAll...) - env, err := testutils.BuildEnv(s.Assert.ResourceRefs) + env, err := expressions.BuildEnv(s.Assert.ResourceRefs) if err != nil { return fmt.Errorf("failed to load TestAssert object from %s: %w", file, err) } @@ -577,7 +592,7 @@ func (s *Step) LoadYAML(file string) error { } for _, assertion := range assertions { - if prg, err := assertion.BuildProgram(env); err != nil { + if prg, err := expressions.BuildProgram(assertion.CELExpression, env); err != nil { errs = append(errs, err) } else { s.Programs[assertion.CELExpression] = prg diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 4f2d0a24..611f078c 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -58,7 +58,6 @@ import ( "github.com/kudobuilder/kuttl/pkg/apis" harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" "github.com/kudobuilder/kuttl/pkg/env" - "github.com/kudobuilder/kuttl/pkg/k8s" ) // ensure that we only add to the scheme once. @@ -1114,7 +1113,7 @@ func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd kuttlENV := make(map[string]string) kuttlENV["NAMESPACE"] = namespace - kuttlENV["KUBECONFIG"] = KubeconfigPath(actualDir, kubeconfigOverride) + kuttlENV["KUBECONFIG"] = kubeconfigPath(actualDir, kubeconfigOverride) kuttlENV["PATH"] = fmt.Sprintf("%s/bin/:%s", actualDir, os.Getenv("PATH")) // by default testsuite timeout is the command timeout @@ -1183,7 +1182,7 @@ func RunCommand(ctx context.Context, namespace string, cmd harness.Command, cwd return nil, nil } -func KubeconfigPath(actualDir, override string) string { +func kubeconfigPath(actualDir, override string) string { if override != "" { if filepath.IsAbs(override) { return override @@ -1221,32 +1220,16 @@ func RunAssertCommands(ctx context.Context, logger Logger, namespace string, com // RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions func RunAssertExpressions( - ctx context.Context, - cl client.Client, programs map[string]cel.Program, - resourceRefs []harness.TestResourceRef, + variables map[string]interface{}, assertAny, assertAll []*harness.Assertion, ) []error { - errs := []error{} + var errs []error if len(assertAny) == 0 && len(assertAll) == 0 { return errs } - variables := make(map[string]interface{}) - for _, resourceRef := range resourceRefs { - namespacedName, referencedResource := resourceRef.BuildResourceReference() - if err := cl.Get( - ctx, - namespacedName, - referencedResource, - ); err != nil { - return []error{fmt.Errorf("failed to get referenced resource '%v': %w", namespacedName, err)} - } - - variables[resourceRef.Ref] = referencedResource.Object - } - var anyExpressionsEvaluation, allExpressionsEvaluation []error for _, expr := range assertAny { prg, ok := programs[expr.CELExpression] @@ -1393,16 +1376,3 @@ func Kubeconfig(cfg *rest.Config, w io.Writer) error { }, }, w) } - -func NewClient(kubeconfig, context string) func(bool) (client.Client, error) { - return func(bool) (client.Client, error) { - config, err := k8s.BuildConfigWithContext(kubeconfig, context) - if err != nil { - return nil, err - } - - return NewRetryClient(config, client.Options{ - Scheme: Scheme(), - }) - } -} diff --git a/pkg/test/utils/kubernetes_test.go b/pkg/test/utils/kubernetes_test.go index a464d4e4..1adcecf3 100644 --- a/pkg/test/utils/kubernetes_test.go +++ b/pkg/test/utils/kubernetes_test.go @@ -134,7 +134,7 @@ func TestKubeconfigPath(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - result := KubeconfigPath(tt.path, tt.override) + result := kubeconfigPath(tt.path, tt.override) assert.Equal(t, tt.expected, result) }) } From 32ec71591a3c679ac4906323dd974c7a385e7b9e Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Tue, 3 Dec 2024 22:39:39 +0530 Subject: [PATCH 11/17] chore: move RunAssertExpressions() to pkg/expressions Signed-off-by: Kumar Mallikarjuna --- pkg/expressions/cel.go | 55 ++++++++++++++++++++++++++++++++++++ pkg/test/step.go | 2 +- pkg/test/utils/kubernetes.go | 55 ------------------------------------ 3 files changed, 56 insertions(+), 56 deletions(-) diff --git a/pkg/expressions/cel.go b/pkg/expressions/cel.go index cd52fe7d..55fb23b7 100644 --- a/pkg/expressions/cel.go +++ b/pkg/expressions/cel.go @@ -1,6 +1,7 @@ package expressions import ( + "errors" "fmt" "github.com/google/cel-go/cel" @@ -37,3 +38,57 @@ func BuildEnv(resourceRefs []v1beta1.TestResourceRef) (*cel.Env, error) { return env, nil } + +// RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions +func RunAssertExpressions( + programs map[string]cel.Program, + variables map[string]interface{}, + assertAny, + assertAll []*v1beta1.Assertion, +) []error { + var errs []error + if len(assertAny) == 0 && len(assertAll) == 0 { + return errs + } + + var anyExpressionsEvaluation, allExpressionsEvaluation []error + for _, expr := range assertAny { + prg, ok := programs[expr.CELExpression] + if !ok { + return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} + } + out, _, err := prg.Eval(variables) + if err != nil { + return []error{fmt.Errorf("failed to evaluate program: %w", err)} + } + + if out.Value() != true { + anyExpressionsEvaluation = append(anyExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) + } + } + + for _, expr := range assertAll { + prg, ok := programs[expr.CELExpression] + if !ok { + return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} + } + out, _, err := prg.Eval(variables) + if err != nil { + return []error{fmt.Errorf("failed to evaluate program: %w", err)} + } + + if out.Value() != true { + allExpressionsEvaluation = append(allExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) + } + } + + if len(assertAny) != 0 && len(anyExpressionsEvaluation) == len(assertAny) { + errs = append(errs, fmt.Errorf("no expression evaluated to true: %w", errors.Join(anyExpressionsEvaluation...))) + } + + if len(allExpressionsEvaluation) != len(assertAll) { + errs = append(errs, fmt.Errorf("not all expressions evaluated to true: %w", errors.Join(allExpressionsEvaluation...))) + } + + return errs +} diff --git a/pkg/test/step.go b/pkg/test/step.go index d8f2693a..1c44ffa4 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -441,7 +441,7 @@ func (s *Step) CheckAssertExpressions( variables[resourceRef.Ref] = referencedResource.Object } - return testutils.RunAssertExpressions(s.Programs, variables, assertAny, assertAll) + return expressions.RunAssertExpressions(s.Programs, variables, assertAny, assertAll) } // Check checks if the resources defined in Asserts and Errors are in the correct state. diff --git a/pkg/test/utils/kubernetes.go b/pkg/test/utils/kubernetes.go index 611f078c..4c703691 100644 --- a/pkg/test/utils/kubernetes.go +++ b/pkg/test/utils/kubernetes.go @@ -19,7 +19,6 @@ import ( "testing" "time" - "github.com/google/cel-go/cel" "github.com/google/shlex" "github.com/pmezard/go-difflib/difflib" "github.com/spf13/pflag" @@ -1218,60 +1217,6 @@ func RunAssertCommands(ctx context.Context, logger Logger, namespace string, com return RunCommands(ctx, logger, namespace, convertAssertCommand(commands, timeout), workdir, timeout, kubeconfigOverride) } -// RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions -func RunAssertExpressions( - programs map[string]cel.Program, - variables map[string]interface{}, - assertAny, - assertAll []*harness.Assertion, -) []error { - var errs []error - if len(assertAny) == 0 && len(assertAll) == 0 { - return errs - } - - var anyExpressionsEvaluation, allExpressionsEvaluation []error - for _, expr := range assertAny { - prg, ok := programs[expr.CELExpression] - if !ok { - return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} - } - out, _, err := prg.Eval(variables) - if err != nil { - return []error{fmt.Errorf("failed to evaluate program: %w", err)} - } - - if out.Value() != true { - anyExpressionsEvaluation = append(anyExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) - } - } - - for _, expr := range assertAll { - prg, ok := programs[expr.CELExpression] - if !ok { - return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} - } - out, _, err := prg.Eval(variables) - if err != nil { - return []error{fmt.Errorf("failed to evaluate program: %w", err)} - } - - if out.Value() != true { - allExpressionsEvaluation = append(allExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) - } - } - - if len(assertAny) != 0 && len(anyExpressionsEvaluation) == len(assertAny) { - errs = append(errs, fmt.Errorf("no expression evaluated to true: %w", errors.Join(anyExpressionsEvaluation...))) - } - - if len(allExpressionsEvaluation) != len(assertAll) { - errs = append(errs, fmt.Errorf("not all expressions evaluated to true: %w", errors.Join(allExpressionsEvaluation...))) - } - - return errs -} - // RunCommands runs a set of commands, returning any errors. // If any (non-background) command fails, the following commands are skipped // commands running in the background are returned From 65dd8134d583c77c76ad208c7e84713c9599b949 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Tue, 3 Dec 2024 22:49:22 +0530 Subject: [PATCH 12/17] refactor: move CEL program loading to a dedicated function Signed-off-by: Kumar Mallikarjuna --- pkg/expressions/cel.go | 49 ++++++++++++++++++++++++++++++++++++++---- pkg/test/step.go | 35 ++---------------------------- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/pkg/expressions/cel.go b/pkg/expressions/cel.go index 55fb23b7..65558281 100644 --- a/pkg/expressions/cel.go +++ b/pkg/expressions/cel.go @@ -6,10 +6,11 @@ import ( "github.com/google/cel-go/cel" - "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" + harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" + "github.com/kudobuilder/kuttl/pkg/test" ) -func BuildProgram(expr string, env *cel.Env) (cel.Program, error) { +func buildProgram(expr string, env *cel.Env) (cel.Program, error) { ast, issues := env.Compile(expr) if issues != nil && issues.Err() != nil { return nil, fmt.Errorf("type-check error: %s", issues.Err()) @@ -23,7 +24,7 @@ func BuildProgram(expr string, env *cel.Env) (cel.Program, error) { return prg, nil } -func BuildEnv(resourceRefs []v1beta1.TestResourceRef) (*cel.Env, error) { +func buildEnv(resourceRefs []harness.TestResourceRef) (*cel.Env, error) { env, err := cel.NewEnv() if err != nil { return nil, fmt.Errorf("failed to create environment: %w", err) @@ -44,7 +45,7 @@ func RunAssertExpressions( programs map[string]cel.Program, variables map[string]interface{}, assertAny, - assertAll []*v1beta1.Assertion, + assertAll []*harness.Assertion, ) []error { var errs []error if len(assertAny) == 0 && len(assertAll) == 0 { @@ -92,3 +93,43 @@ func RunAssertExpressions( return errs } + +func LoadPrograms(s *test.Step) error { + var errs []error + for _, resourceRef := range s.Assert.ResourceRefs { + if err := resourceRef.Validate(); err != nil { + errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err)) + } + } + + if len(errs) > 0 { + return fmt.Errorf("failed to load resource reference(s): %w", errors.Join(errs...)) + } + + var assertions []*harness.Assertion + assertions = append(assertions, s.Assert.AssertAny...) + assertions = append(assertions, s.Assert.AssertAll...) + + env, err := buildEnv(s.Assert.ResourceRefs) + if err != nil { + return fmt.Errorf("failed to build environment: %w", err) + } + + if len(assertions) > 0 { + s.Programs = make(map[string]cel.Program) + } + + for _, assertion := range assertions { + if prg, err := buildProgram(assertion.CELExpression, env); err != nil { + errs = append(errs, err) + } else { + s.Programs[assertion.CELExpression] = prg + } + } + + if len(errs) > 0 { + return fmt.Errorf("failed to build program(s): %w", errors.Join(errs...)) + } + + return nil +} diff --git a/pkg/test/step.go b/pkg/test/step.go index 1c44ffa4..3797bca6 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -567,40 +567,9 @@ func (s *Step) LoadYAML(file string) error { return fmt.Errorf("failed to load TestAssert object from %s: it contains an object of type %T", file, obj) } - var errs []error - for _, resourceRef := range s.Assert.ResourceRefs { - if err := resourceRef.Validate(); err != nil { - errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err)) - } - } - - if len(errs) > 0 { - return fmt.Errorf("failed to load TestAssert object from %s: %w", file, errors.Join(errs...)) - } - - var assertions []*harness.Assertion - assertions = append(assertions, s.Assert.AssertAny...) - assertions = append(assertions, s.Assert.AssertAll...) - - env, err := expressions.BuildEnv(s.Assert.ResourceRefs) + err := expressions.LoadPrograms(s) if err != nil { - return fmt.Errorf("failed to load TestAssert object from %s: %w", file, err) - } - - if len(assertions) > 0 { - s.Programs = make(map[string]cel.Program) - } - - for _, assertion := range assertions { - if prg, err := expressions.BuildProgram(assertion.CELExpression, env); err != nil { - errs = append(errs, err) - } else { - s.Programs[assertion.CELExpression] = prg - } - } - - if len(errs) > 0 { - return fmt.Errorf("failed to load TestAssert object from %s: %w", file, errors.Join(errs...)) + return fmt.Errorf("failed to load programs: %w", err) } } else { asserts = append(asserts, obj) From 79c6f842568cc49e500cdf85b28a5c6ba0e8e280 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Tue, 3 Dec 2024 22:53:06 +0530 Subject: [PATCH 13/17] refactor: move program-loading to Step out of LoadPrograms() Signed-off-by: Kumar Mallikarjuna --- pkg/expressions/cel.go | 24 ++++++++++++------------ pkg/test/step.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/expressions/cel.go b/pkg/expressions/cel.go index 65558281..e25a331c 100644 --- a/pkg/expressions/cel.go +++ b/pkg/expressions/cel.go @@ -7,7 +7,6 @@ import ( "github.com/google/cel-go/cel" harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" - "github.com/kudobuilder/kuttl/pkg/test" ) func buildProgram(expr string, env *cel.Env) (cel.Program, error) { @@ -94,42 +93,43 @@ func RunAssertExpressions( return errs } -func LoadPrograms(s *test.Step) error { +func LoadPrograms(testAssert *harness.TestAssert) (map[string]cel.Program, error) { var errs []error - for _, resourceRef := range s.Assert.ResourceRefs { + for _, resourceRef := range testAssert.ResourceRefs { if err := resourceRef.Validate(); err != nil { errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err)) } } if len(errs) > 0 { - return fmt.Errorf("failed to load resource reference(s): %w", errors.Join(errs...)) + return nil, fmt.Errorf("failed to load resource reference(s): %w", errors.Join(errs...)) } var assertions []*harness.Assertion - assertions = append(assertions, s.Assert.AssertAny...) - assertions = append(assertions, s.Assert.AssertAll...) + assertions = append(assertions, testAssert.AssertAny...) + assertions = append(assertions, testAssert.AssertAll...) - env, err := buildEnv(s.Assert.ResourceRefs) + env, err := buildEnv(testAssert.ResourceRefs) if err != nil { - return fmt.Errorf("failed to build environment: %w", err) + return nil, fmt.Errorf("failed to build environment: %w", err) } + var programs map[string]cel.Program if len(assertions) > 0 { - s.Programs = make(map[string]cel.Program) + programs = make(map[string]cel.Program) } for _, assertion := range assertions { if prg, err := buildProgram(assertion.CELExpression, env); err != nil { errs = append(errs, err) } else { - s.Programs[assertion.CELExpression] = prg + programs[assertion.CELExpression] = prg } } if len(errs) > 0 { - return fmt.Errorf("failed to build program(s): %w", errors.Join(errs...)) + return nil, fmt.Errorf("failed to build program(s): %w", errors.Join(errs...)) } - return nil + return programs, nil } diff --git a/pkg/test/step.go b/pkg/test/step.go index 3797bca6..08db8231 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -567,7 +567,7 @@ func (s *Step) LoadYAML(file string) error { return fmt.Errorf("failed to load TestAssert object from %s: it contains an object of type %T", file, obj) } - err := expressions.LoadPrograms(s) + s.Programs, err = expressions.LoadPrograms(s.Assert) if err != nil { return fmt.Errorf("failed to load programs: %w", err) } From 8ac7cb1db142216dd1ad125384a77bf3cedc8f5d Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 4 Dec 2024 20:42:03 +0530 Subject: [PATCH 14/17] chore: add tests for `TestResourceRef` Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 15 +- .../testharness/v1beta1/expression_test.go | 181 ++++++++++++++++++ 2 files changed, 192 insertions(+), 4 deletions(-) create mode 100644 pkg/apis/testharness/v1beta1/expression_test.go diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go index de210a2d..2209c074 100644 --- a/pkg/apis/testharness/v1beta1/expression.go +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -10,6 +10,13 @@ import ( "k8s.io/apimachinery/pkg/types" ) +var ( + apiVersionInvalidErr = errors.New("apiVersion not of the format (/)") + kindNotSpecifiedErr = errors.New("kind not specified") + nameNotSpecifiedErr = errors.New("name not specified") + refNotSpecifiedErr = errors.New("ref not specified") +) + func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) { referencedResource = &unstructured.Unstructured{} apiVersionSplit := strings.Split(t.APIVersion, "/") @@ -34,13 +41,13 @@ func (t *TestResourceRef) Validate() error { apiVersionSplit := strings.Split(t.APIVersion, "/") switch { case t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2): - return fmt.Errorf("apiVersion '%v' not of the format (/)", t.APIVersion) + return apiVersionInvalidErr case t.Kind == "": - return errors.New("kind not specified") + return kindNotSpecifiedErr case t.Name == "": - return errors.New("name not specified") + return nameNotSpecifiedErr case t.Ref == "": - return errors.New("ref not specified") + return refNotSpecifiedErr } return nil diff --git a/pkg/apis/testharness/v1beta1/expression_test.go b/pkg/apis/testharness/v1beta1/expression_test.go new file mode 100644 index 00000000..b69b6b00 --- /dev/null +++ b/pkg/apis/testharness/v1beta1/expression_test.go @@ -0,0 +1,181 @@ +package v1beta1 + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +func TestValidate(t *testing.T) { + testCases := []struct { + name string + testResourceRef TestResourceRef + errored bool + expectedError error + }{ + { + name: "apiVersion is not specified", + testResourceRef: TestResourceRef{ + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: true, + expectedError: apiVersionInvalidErr, + }, + { + name: "apiVersion is invalid", + testResourceRef: TestResourceRef{ + APIVersion: "x/y/z", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: true, + expectedError: apiVersionInvalidErr, + }, + { + name: "apiVersion is valid and group is vacuous", + testResourceRef: TestResourceRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: false, + }, + { + name: "apiVersion has both group name and version", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: false, + }, + { + name: "kind is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: true, + expectedError: kindNotSpecifiedErr, + }, + { + name: "name is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Ref: "testDeployment", + }, + errored: true, + expectedError: nameNotSpecifiedErr, + }, + { + name: "ref is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + }, + errored: true, + expectedError: refNotSpecifiedErr, + }, + { + name: "all attributes are present and valid", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.testResourceRef.Validate() + if !tc.errored { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tc.expectedError) + } + }) + } +} + +func TestBuildResourceReference(t *testing.T) { + buildObject := func(gvk schema.GroupVersionKind) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + return obj + } + + testCases := []struct { + name string + testResourceRef TestResourceRef + namespacedName types.NamespacedName + resourceReference *unstructured.Unstructured + }{ + { + name: "group name is vacuous", + testResourceRef: TestResourceRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + namespacedName: types.NamespacedName{ + Namespace: "test", + Name: "test-pod", + }, + resourceReference: buildObject(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}), + }, + { + name: "group name is present", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + namespacedName: types.NamespacedName{ + Namespace: "test", + Name: "test-deployment", + }, + resourceReference: buildObject(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + namspacedName, referencedResource := tc.testResourceRef.BuildResourceReference() + assert.Equal(t, tc.namespacedName, namspacedName) + assert.True( + t, + reflect.DeepEqual(tc.resourceReference, referencedResource), + "constructed unstructured reference does not match, expected '%s', got '%s'", + tc.resourceReference, + referencedResource, + ) + }) + } +} From a4ebceff1398a9c33ab471bbcd37b3da7826d251 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 4 Dec 2024 22:05:57 +0530 Subject: [PATCH 15/17] fix: correct evaluation for `assertAll` Signed-off-by: Kumar Mallikarjuna --- pkg/expressions/cel.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/expressions/cel.go b/pkg/expressions/cel.go index e25a331c..711ff028 100644 --- a/pkg/expressions/cel.go +++ b/pkg/expressions/cel.go @@ -86,7 +86,7 @@ func RunAssertExpressions( errs = append(errs, fmt.Errorf("no expression evaluated to true: %w", errors.Join(anyExpressionsEvaluation...))) } - if len(allExpressionsEvaluation) != len(assertAll) { + if len(allExpressionsEvaluation) > 0 { errs = append(errs, fmt.Errorf("not all expressions evaluated to true: %w", errors.Join(allExpressionsEvaluation...))) } From f2422aeaf0db7c59970ec9751f1ee8ff359feb6f Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 4 Dec 2024 23:16:30 +0530 Subject: [PATCH 16/17] chore: add integration tests for expressions Signed-off-by: Kumar Mallikarjuna --- pkg/test/expression_integration_test.go | 118 ++++++++++++++++++ .../check_deployment_name/00-assert.yaml | 11 ++ .../00-assert.yaml | 11 ++ .../check_multiple_assert_all/00-assert.yaml | 17 +++ .../00-assert.yaml | 17 +++ .../check_multiple_assert_any/00-assert.yaml | 17 +++ .../00-assert.yaml | 17 +++ .../invalid_expression/00-assert.yaml | 11 ++ 8 files changed, 219 insertions(+) create mode 100644 pkg/test/expression_integration_test.go create mode 100644 pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml create mode 100644 pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml create mode 100644 pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml create mode 100644 pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml create mode 100644 pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml create mode 100644 pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml create mode 100644 pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml diff --git a/pkg/test/expression_integration_test.go b/pkg/test/expression_integration_test.go new file mode 100644 index 00000000..529d0c29 --- /dev/null +++ b/pkg/test/expression_integration_test.go @@ -0,0 +1,118 @@ +package test + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + testutils "github.com/kudobuilder/kuttl/pkg/test/utils" +) + +func TestAssertExpressions(t *testing.T) { + codednsDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "coredns", + Namespace: "kube-system", + }, + } + metricServerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-server-xyz-pqr", + Namespace: "kube-system", + Labels: map[string]string{ + "app": "metrics-server", + }, + }, + } + + buildTestStep := func(tcName string) *Step { + return &Step{ + Name: t.Name(), + Index: 0, + Logger: testutils.NewTestLogger(t, tcName), + Client: func(bool) (client.Client, error) { + return fake. + NewClientBuilder(). + WithObjects(codednsDeployment, metricServerPod). + WithScheme(testutils.Scheme()). + Build(), nil + }, + DiscoveryClient: func() (discovery.DiscoveryInterface, error) { + return testutils.FakeDiscoveryClient(), nil + }, + } + } + + testCases := []struct { + name string + loadingFailed bool + runFailed bool + errorMessage string + }{ + { + name: "invalid expression", + loadingFailed: true, + errorMessage: "undeclared reference", + }, + { + name: "check deployment name", + }, + { + name: "check incorrect deployment name", + runFailed: true, + errorMessage: "not all expressions evaluated to true", + }, + { + name: "check multiple assert all", + }, + { + name: "check multiple assert all with one failing", + runFailed: true, + errorMessage: "not all expressions evaluated to true", + }, + { + name: "check multiple assert any", + }, + { + name: "check multiple assert any with all failing", + runFailed: true, + errorMessage: "no expression evaluated to true", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + step := buildTestStep(tc.name) + + fName := fmt.Sprintf( + "step_integration_test_data/assert_expressions/%s/00-assert.yaml", + strings.Replace(tc.name, " ", "_", -1), + ) + + // Load test that has an invalid expression + err := step.LoadYAML(fName) + if !tc.loadingFailed { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.errorMessage) + return + } + + err = errors.Join(step.Run(t, "")...) + if !tc.runFailed { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.errorMessage) + } + }) + } +} diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml new file mode 100644 index 00000000..9b0cda37 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "coredns.metadata.name == 'coredns'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml new file mode 100644 index 00000000..3ec2e308 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "coredns.metadata.name == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml new file mode 100644 index 00000000..64855289 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAll: + - celExpr: "coredns.metadata.name == 'coredns'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml new file mode 100644 index 00000000..7f850071 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAll: + - celExpr: "coredns.metadata.name == 'metrics-server'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml new file mode 100644 index 00000000..64659111 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAny: + - celExpr: "coredns.metadata.name == 'coredns'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server-1.6'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml new file mode 100644 index 00000000..786b600f --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAny: + - celExpr: "coredns.metadata.name == 'metrics-server'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server-1.6'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml new file mode 100644 index 00000000..86a8b392 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "badVariable.metadata.name == 'coredns'" +timeout: 1 From e6b22fc5be6bd0a3038d7d842d1393de38815fe0 Mon Sep 17 00:00:00 2001 From: Kumar Mallikarjuna Date: Wed, 4 Dec 2024 23:46:41 +0530 Subject: [PATCH 17/17] chore: make linter happy Signed-off-by: Kumar Mallikarjuna --- pkg/apis/testharness/v1beta1/expression.go | 16 ++++++++-------- pkg/apis/testharness/v1beta1/expression_test.go | 10 +++++----- pkg/test/expression_integration_test.go | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go index 2209c074..f696ec02 100644 --- a/pkg/apis/testharness/v1beta1/expression.go +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -11,10 +11,10 @@ import ( ) var ( - apiVersionInvalidErr = errors.New("apiVersion not of the format (/)") - kindNotSpecifiedErr = errors.New("kind not specified") - nameNotSpecifiedErr = errors.New("name not specified") - refNotSpecifiedErr = errors.New("ref not specified") + errAPIVersionInvalid = errors.New("apiVersion not of the format (/)") + errKindNotSpecified = errors.New("kind not specified") + errNameNotSpecified = errors.New("name not specified") + errRefNotSpecified = errors.New("ref not specified") ) func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) { @@ -41,13 +41,13 @@ func (t *TestResourceRef) Validate() error { apiVersionSplit := strings.Split(t.APIVersion, "/") switch { case t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2): - return apiVersionInvalidErr + return errAPIVersionInvalid case t.Kind == "": - return kindNotSpecifiedErr + return errKindNotSpecified case t.Name == "": - return nameNotSpecifiedErr + return errNameNotSpecified case t.Ref == "": - return refNotSpecifiedErr + return errRefNotSpecified } return nil diff --git a/pkg/apis/testharness/v1beta1/expression_test.go b/pkg/apis/testharness/v1beta1/expression_test.go index b69b6b00..cac25e4d 100644 --- a/pkg/apis/testharness/v1beta1/expression_test.go +++ b/pkg/apis/testharness/v1beta1/expression_test.go @@ -26,7 +26,7 @@ func TestValidate(t *testing.T) { Ref: "testPod", }, errored: true, - expectedError: apiVersionInvalidErr, + expectedError: errAPIVersionInvalid, }, { name: "apiVersion is invalid", @@ -38,7 +38,7 @@ func TestValidate(t *testing.T) { Ref: "testPod", }, errored: true, - expectedError: apiVersionInvalidErr, + expectedError: errAPIVersionInvalid, }, { name: "apiVersion is valid and group is vacuous", @@ -71,7 +71,7 @@ func TestValidate(t *testing.T) { Ref: "testDeployment", }, errored: true, - expectedError: kindNotSpecifiedErr, + expectedError: errKindNotSpecified, }, { name: "name is not specified", @@ -82,7 +82,7 @@ func TestValidate(t *testing.T) { Ref: "testDeployment", }, errored: true, - expectedError: nameNotSpecifiedErr, + expectedError: errNameNotSpecified, }, { name: "ref is not specified", @@ -93,7 +93,7 @@ func TestValidate(t *testing.T) { Name: "test-deployment", }, errored: true, - expectedError: refNotSpecifiedErr, + expectedError: errRefNotSpecified, }, { name: "all attributes are present and valid", diff --git a/pkg/test/expression_integration_test.go b/pkg/test/expression_integration_test.go index 529d0c29..2dac15bf 100644 --- a/pkg/test/expression_integration_test.go +++ b/pkg/test/expression_integration_test.go @@ -95,7 +95,7 @@ func TestAssertExpressions(t *testing.T) { fName := fmt.Sprintf( "step_integration_test_data/assert_expressions/%s/00-assert.yaml", - strings.Replace(tc.name, " ", "_", -1), + strings.ReplaceAll(tc.name, " ", "_"), ) // Load test that has an invalid expression