diff --git a/go.mod b/go.mod index 9d4927ce..73ba8f23 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/docker/docker v27.3.1+incompatible github.com/dustin/go-humanize v1.0.1 github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 + github.com/google/cel-go v0.20.1 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/spf13/cobra v1.8.1 @@ -30,6 +31,7 @@ require ( github.com/BurntSushi/toml v1.4.0 // indirect github.com/Microsoft/go-winio v0.5.1 // indirect github.com/alessio/shellescape v1.4.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect @@ -70,12 +72,14 @@ require ( github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/stoewer/go-strcase v1.2.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect go.opentelemetry.io/otel/metric v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect golang.org/x/mod v0.21.0 // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect @@ -85,6 +89,8 @@ require ( golang.org/x/text v0.19.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.26.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 805bc0a9..bcec56a7 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0= github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -70,6 +72,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84= +github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -165,12 +169,15 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -255,7 +262,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= diff --git a/pkg/apis/testharness/v1beta1/expression.go b/pkg/apis/testharness/v1beta1/expression.go new file mode 100644 index 00000000..f696ec02 --- /dev/null +++ b/pkg/apis/testharness/v1beta1/expression.go @@ -0,0 +1,65 @@ +package v1beta1 + +import ( + "errors" + "fmt" + "strings" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +var ( + errAPIVersionInvalid = errors.New("apiVersion not of the format (/)") + errKindNotSpecified = errors.New("kind not specified") + errNameNotSpecified = errors.New("name not specified") + errRefNotSpecified = errors.New("ref not specified") +) + +func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) { + referencedResource = &unstructured.Unstructured{} + apiVersionSplit := strings.Split(t.APIVersion, "/") + gvk := schema.GroupVersionKind{ + Version: apiVersionSplit[len(apiVersionSplit)-1], + Kind: t.Kind, + } + if len(apiVersionSplit) > 1 { + gvk.Group = apiVersionSplit[0] + } + referencedResource.SetGroupVersionKind(gvk) + + namespacedName = types.NamespacedName{ + Namespace: t.Namespace, + Name: t.Name, + } + + return +} + +func (t *TestResourceRef) Validate() error { + apiVersionSplit := strings.Split(t.APIVersion, "/") + switch { + case t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2): + return errAPIVersionInvalid + case t.Kind == "": + return errKindNotSpecified + case t.Name == "": + return errNameNotSpecified + case t.Ref == "": + return errRefNotSpecified + } + + return nil +} + +func (t *TestResourceRef) String() string { + return fmt.Sprintf( + "apiVersion=%v, kind=%v, namespace=%v, name=%v, ref=%v", + t.APIVersion, + t.Kind, + t.Namespace, + t.Name, + t.Ref, + ) +} diff --git a/pkg/apis/testharness/v1beta1/expression_test.go b/pkg/apis/testharness/v1beta1/expression_test.go new file mode 100644 index 00000000..cac25e4d --- /dev/null +++ b/pkg/apis/testharness/v1beta1/expression_test.go @@ -0,0 +1,181 @@ +package v1beta1 + +import ( + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +func TestValidate(t *testing.T) { + testCases := []struct { + name string + testResourceRef TestResourceRef + errored bool + expectedError error + }{ + { + name: "apiVersion is not specified", + testResourceRef: TestResourceRef{ + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: true, + expectedError: errAPIVersionInvalid, + }, + { + name: "apiVersion is invalid", + testResourceRef: TestResourceRef{ + APIVersion: "x/y/z", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: true, + expectedError: errAPIVersionInvalid, + }, + { + name: "apiVersion is valid and group is vacuous", + testResourceRef: TestResourceRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + errored: false, + }, + { + name: "apiVersion has both group name and version", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: false, + }, + { + name: "kind is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: true, + expectedError: errKindNotSpecified, + }, + { + name: "name is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Ref: "testDeployment", + }, + errored: true, + expectedError: errNameNotSpecified, + }, + { + name: "ref is not specified", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + }, + errored: true, + expectedError: errRefNotSpecified, + }, + { + name: "all attributes are present and valid", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + errored: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.testResourceRef.Validate() + if !tc.errored { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, tc.expectedError) + } + }) + } +} + +func TestBuildResourceReference(t *testing.T) { + buildObject := func(gvk schema.GroupVersionKind) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + return obj + } + + testCases := []struct { + name string + testResourceRef TestResourceRef + namespacedName types.NamespacedName + resourceReference *unstructured.Unstructured + }{ + { + name: "group name is vacuous", + testResourceRef: TestResourceRef{ + APIVersion: "v1", + Kind: "Pod", + Namespace: "test", + Name: "test-pod", + Ref: "testPod", + }, + namespacedName: types.NamespacedName{ + Namespace: "test", + Name: "test-pod", + }, + resourceReference: buildObject(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}), + }, + { + name: "group name is present", + testResourceRef: TestResourceRef{ + APIVersion: "apps/v1", + Kind: "Deployment", + Namespace: "test", + Name: "test-deployment", + Ref: "testDeployment", + }, + namespacedName: types.NamespacedName{ + Namespace: "test", + Name: "test-deployment", + }, + resourceReference: buildObject(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + namspacedName, referencedResource := tc.testResourceRef.BuildResourceReference() + assert.Equal(t, tc.namespacedName, namspacedName) + assert.True( + t, + reflect.DeepEqual(tc.resourceReference, referencedResource), + "constructed unstructured reference does not match, expected '%s', got '%s'", + tc.resourceReference, + referencedResource, + ) + }) + } +} diff --git a/pkg/apis/testharness/v1beta1/test_types.go b/pkg/apis/testharness/v1beta1/test_types.go index 28d3c518..6e2f1cd8 100644 --- a/pkg/apis/testharness/v1beta1/test_types.go +++ b/pkg/apis/testharness/v1beta1/test_types.go @@ -152,6 +152,11 @@ type TestAssert struct { Collectors []*TestCollector `json:"collectors,omitempty"` // Commands is a set of commands to be run as assertions for the current step Commands []TestAssertCommand `json:"commands,omitempty"` + + ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"` + + AssertAny []*Assertion `json:"assertAny,omitempty"` + AssertAll []*Assertion `json:"assertAll,omitempty"` } // TestAssertCommand an assertion based on the result of the execution of a command @@ -222,6 +227,18 @@ type TestCollector struct { Cmd string `json:"command,omitempty"` } +type TestResourceRef struct { + APIVersion string `json:"apiVersion,omitempty"` + Kind string `json:"kind,omitempty"` + Namespace string `json:"namespace,omitempty"` + Name string `json:"name,omitempty"` + Ref string `json:"ref,omitempty"` +} + +type Assertion struct { + CELExpression string `json:"celExpr,omitempty"` +} + // DefaultKINDContext defines the default kind context to use. const DefaultKINDContext = "kind" diff --git a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go index 4698a0c6..303263f4 100644 --- a/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/testharness/v1beta1/zz_generated.deepcopy.go @@ -24,6 +24,22 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Assertion) DeepCopyInto(out *Assertion) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Assertion. +func (in *Assertion) DeepCopy() *Assertion { + if in == nil { + return nil + } + out := new(Assertion) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Command) DeepCopyInto(out *Command) { *out = *in @@ -95,6 +111,33 @@ func (in *TestAssert) DeepCopyInto(out *TestAssert) { *out = make([]TestAssertCommand, len(*in)) copy(*out, *in) } + if in.ResourceRefs != nil { + in, out := &in.ResourceRefs, &out.ResourceRefs + *out = make([]TestResourceRef, len(*in)) + copy(*out, *in) + } + if in.AssertAny != nil { + in, out := &in.AssertAny, &out.AssertAny + *out = make([]*Assertion, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Assertion) + **out = **in + } + } + } + if in.AssertAll != nil { + in, out := &in.AssertAll, &out.AssertAll + *out = make([]*Assertion, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(Assertion) + **out = **in + } + } + } return } @@ -179,6 +222,22 @@ func (in *TestFile) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TestResourceRef) DeepCopyInto(out *TestResourceRef) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TestResourceRef. +func (in *TestResourceRef) DeepCopy() *TestResourceRef { + if in == nil { + return nil + } + out := new(TestResourceRef) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TestStep) DeepCopyInto(out *TestStep) { *out = *in diff --git a/pkg/expressions/cel.go b/pkg/expressions/cel.go new file mode 100644 index 00000000..711ff028 --- /dev/null +++ b/pkg/expressions/cel.go @@ -0,0 +1,135 @@ +package expressions + +import ( + "errors" + "fmt" + + "github.com/google/cel-go/cel" + + harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" +) + +func buildProgram(expr string, env *cel.Env) (cel.Program, error) { + ast, issues := env.Compile(expr) + if issues != nil && issues.Err() != nil { + return nil, fmt.Errorf("type-check error: %s", issues.Err()) + } + + prg, err := env.Program(ast) + if err != nil { + return nil, fmt.Errorf("program construction error: %w", err) + } + + return prg, nil +} + +func buildEnv(resourceRefs []harness.TestResourceRef) (*cel.Env, error) { + env, err := cel.NewEnv() + if err != nil { + return nil, fmt.Errorf("failed to create environment: %w", err) + } + + for _, resourceRef := range resourceRefs { + env, err = env.Extend(cel.Variable(resourceRef.Ref, cel.DynType)) + if err != nil { + return nil, fmt.Errorf("failed to add resource parameter '%v' to environment: %w", resourceRef.Ref, err) + } + } + + return env, nil +} + +// RunAssertExpressions evaluates a set of CEL expressions specified as AnyAllExpressions +func RunAssertExpressions( + programs map[string]cel.Program, + variables map[string]interface{}, + assertAny, + assertAll []*harness.Assertion, +) []error { + var errs []error + if len(assertAny) == 0 && len(assertAll) == 0 { + return errs + } + + var anyExpressionsEvaluation, allExpressionsEvaluation []error + for _, expr := range assertAny { + prg, ok := programs[expr.CELExpression] + if !ok { + return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} + } + out, _, err := prg.Eval(variables) + if err != nil { + return []error{fmt.Errorf("failed to evaluate program: %w", err)} + } + + if out.Value() != true { + anyExpressionsEvaluation = append(anyExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) + } + } + + for _, expr := range assertAll { + prg, ok := programs[expr.CELExpression] + if !ok { + return []error{fmt.Errorf("couldn't find pre-built program for expression: %v", expr.CELExpression)} + } + out, _, err := prg.Eval(variables) + if err != nil { + return []error{fmt.Errorf("failed to evaluate program: %w", err)} + } + + if out.Value() != true { + allExpressionsEvaluation = append(allExpressionsEvaluation, fmt.Errorf("expression '%v' evaluated to '%v'", expr.CELExpression, out.Value())) + } + } + + if len(assertAny) != 0 && len(anyExpressionsEvaluation) == len(assertAny) { + errs = append(errs, fmt.Errorf("no expression evaluated to true: %w", errors.Join(anyExpressionsEvaluation...))) + } + + if len(allExpressionsEvaluation) > 0 { + errs = append(errs, fmt.Errorf("not all expressions evaluated to true: %w", errors.Join(allExpressionsEvaluation...))) + } + + return errs +} + +func LoadPrograms(testAssert *harness.TestAssert) (map[string]cel.Program, error) { + var errs []error + for _, resourceRef := range testAssert.ResourceRefs { + if err := resourceRef.Validate(); err != nil { + errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err)) + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to load resource reference(s): %w", errors.Join(errs...)) + } + + var assertions []*harness.Assertion + assertions = append(assertions, testAssert.AssertAny...) + assertions = append(assertions, testAssert.AssertAll...) + + env, err := buildEnv(testAssert.ResourceRefs) + if err != nil { + return nil, fmt.Errorf("failed to build environment: %w", err) + } + + var programs map[string]cel.Program + if len(assertions) > 0 { + programs = make(map[string]cel.Program) + } + + for _, assertion := range assertions { + if prg, err := buildProgram(assertion.CELExpression, env); err != nil { + errs = append(errs, err) + } else { + programs[assertion.CELExpression] = prg + } + } + + if len(errs) > 0 { + return nil, fmt.Errorf("failed to build program(s): %w", errors.Join(errs...)) + } + + return programs, nil +} diff --git a/pkg/test/expression_integration_test.go b/pkg/test/expression_integration_test.go new file mode 100644 index 00000000..2dac15bf --- /dev/null +++ b/pkg/test/expression_integration_test.go @@ -0,0 +1,118 @@ +package test + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/discovery" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + testutils "github.com/kudobuilder/kuttl/pkg/test/utils" +) + +func TestAssertExpressions(t *testing.T) { + codednsDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "coredns", + Namespace: "kube-system", + }, + } + metricServerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "metrics-server-xyz-pqr", + Namespace: "kube-system", + Labels: map[string]string{ + "app": "metrics-server", + }, + }, + } + + buildTestStep := func(tcName string) *Step { + return &Step{ + Name: t.Name(), + Index: 0, + Logger: testutils.NewTestLogger(t, tcName), + Client: func(bool) (client.Client, error) { + return fake. + NewClientBuilder(). + WithObjects(codednsDeployment, metricServerPod). + WithScheme(testutils.Scheme()). + Build(), nil + }, + DiscoveryClient: func() (discovery.DiscoveryInterface, error) { + return testutils.FakeDiscoveryClient(), nil + }, + } + } + + testCases := []struct { + name string + loadingFailed bool + runFailed bool + errorMessage string + }{ + { + name: "invalid expression", + loadingFailed: true, + errorMessage: "undeclared reference", + }, + { + name: "check deployment name", + }, + { + name: "check incorrect deployment name", + runFailed: true, + errorMessage: "not all expressions evaluated to true", + }, + { + name: "check multiple assert all", + }, + { + name: "check multiple assert all with one failing", + runFailed: true, + errorMessage: "not all expressions evaluated to true", + }, + { + name: "check multiple assert any", + }, + { + name: "check multiple assert any with all failing", + runFailed: true, + errorMessage: "no expression evaluated to true", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + step := buildTestStep(tc.name) + + fName := fmt.Sprintf( + "step_integration_test_data/assert_expressions/%s/00-assert.yaml", + strings.ReplaceAll(tc.name, " ", "_"), + ) + + // Load test that has an invalid expression + err := step.LoadYAML(fName) + if !tc.loadingFailed { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.errorMessage) + return + } + + err = errors.Join(step.Run(t, "")...) + if !tc.runFailed { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tc.errorMessage) + } + }) + } +} diff --git a/pkg/test/step.go b/pkg/test/step.go index 2e722f3a..08db8231 100644 --- a/pkg/test/step.go +++ b/pkg/test/step.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/google/cel-go/cel" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -23,6 +24,7 @@ import ( harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1" "github.com/kudobuilder/kuttl/pkg/env" + "github.com/kudobuilder/kuttl/pkg/expressions" kfile "github.com/kudobuilder/kuttl/pkg/file" "github.com/kudobuilder/kuttl/pkg/http" testutils "github.com/kudobuilder/kuttl/pkg/test/utils" @@ -45,6 +47,8 @@ type Step struct { Step *harness.TestStep Assert *harness.TestAssert + Programs map[string]cel.Program + Asserts []client.Object Apply []client.Object Errors []client.Object @@ -412,6 +416,34 @@ func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, comman return testErrors } +func (s *Step) CheckAssertExpressions( + ctx context.Context, + resourceRefs []harness.TestResourceRef, + assertAny, + assertAll []*harness.Assertion, +) []error { + client, err := s.Client(false) + if err != nil { + return []error{err} + } + + variables := make(map[string]interface{}) + for _, resourceRef := range resourceRefs { + namespacedName, referencedResource := resourceRef.BuildResourceReference() + if err := client.Get( + ctx, + namespacedName, + referencedResource, + ); err != nil { + return []error{fmt.Errorf("failed to get referenced resource '%v': %w", namespacedName, err)} + } + + variables[resourceRef.Ref] = referencedResource.Object + } + + return expressions.RunAssertExpressions(s.Programs, variables, assertAny, assertAll) +} + // Check checks if the resources defined in Asserts and Errors are in the correct state. func (s *Step) Check(namespace string, timeout int) []error { testErrors := []error{} @@ -422,6 +454,7 @@ func (s *Step) Check(namespace string, timeout int) []error { if s.Assert != nil { testErrors = append(testErrors, s.CheckAssertCommands(context.TODO(), namespace, s.Assert.Commands, timeout)...) + testErrors = append(testErrors, s.CheckAssertExpressions(context.TODO(), s.Assert.ResourceRefs, s.Assert.AssertAny, s.Assert.AssertAll)...) } for _, expected := range s.Errors { @@ -533,6 +566,11 @@ func (s *Step) LoadYAML(file string) error { } else { return fmt.Errorf("failed to load TestAssert object from %s: it contains an object of type %T", file, obj) } + + s.Programs, err = expressions.LoadPrograms(s.Assert) + if err != nil { + return fmt.Errorf("failed to load programs: %w", err) + } } else { asserts = append(asserts, obj) } diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml new file mode 100644 index 00000000..9b0cda37 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_deployment_name/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "coredns.metadata.name == 'coredns'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml new file mode 100644 index 00000000..3ec2e308 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_incorrect_deployment_name/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "coredns.metadata.name == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml new file mode 100644 index 00000000..64855289 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAll: + - celExpr: "coredns.metadata.name == 'coredns'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml new file mode 100644 index 00000000..7f850071 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_all_with_one_failing/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAll: + - celExpr: "coredns.metadata.name == 'metrics-server'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml new file mode 100644 index 00000000..64659111 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAny: + - celExpr: "coredns.metadata.name == 'coredns'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server-1.6'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml new file mode 100644 index 00000000..786b600f --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/check_multiple_assert_any_with_all_failing/00-assert.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns + - apiVersion: v1 + kind: Pod + namespace: kube-system + name: metrics-server-xyz-pqr + ref: metricsServer +assertAny: + - celExpr: "coredns.metadata.name == 'metrics-server'" + - celExpr: "metricsServer.metadata.labels['app'] == 'metrics-server-1.6'" +timeout: 1 diff --git a/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml b/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml new file mode 100644 index 00000000..86a8b392 --- /dev/null +++ b/pkg/test/step_integration_test_data/assert_expressions/invalid_expression/00-assert.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +resourceRefs: + - apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + name: coredns + ref: coredns +assertAll: + - celExpr: "badVariable.metadata.name == 'coredns'" +timeout: 1