[KEP-0009] feat: add expression based assertions

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/docker/docker v27.3.1+incompatible
 	github.com/dustin/go-humanize v1.0.1
 	github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0
+	github.com/google/cel-go v0.20.1
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2
 	github.com/spf13/cobra v1.8.1
@@ -30,6 +31,7 @@ require (
 	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/Microsoft/go-winio v0.5.1 // indirect
 	github.com/alessio/shellescape v1.4.2 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
 	github.com/containerd/log v0.1.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/distribution/reference v0.6.0 // indirect
@@ -70,12 +72,14 @@ require (
 	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/stoewer/go-strcase v1.2.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.27.0 // indirect
 	go.opentelemetry.io/otel/metric v1.28.0 // indirect
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
 	golang.org/x/mod v0.21.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/oauth2 v0.21.0 // indirect
@@ -85,6 +89,8 @@ require (
 	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.26.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/alessio/shellescape v1.4.2 h1:MHPfaU+ddJ0/bYWpgIeUnQUqKrlJ1S7BfEYPM4uEoM0=
 github.com/alessio/shellescape v1.4.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
+github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI=
+github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
@@ -70,6 +72,8 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l
 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
 github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/cel-go v0.20.1 h1:nDx9r8S3L4pE61eDdt8igGj8rf5kjYR3ILxWIpWNi84=
+github.com/google/cel-go v0.20.1/go.mod h1:kWcIzTsPX0zmQ+H3TirHstLLf9ep5QTsZBN9u4dOYLg=
 github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
 github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -165,12 +169,15 @@ github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3k
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
@@ -255,7 +262,6 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw=
 gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY=
-google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
 google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw=
 google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA=

pkg/apis/testharness/v1beta1/expression.go

@@ -0,0 +1,75 @@
+package v1beta1
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/google/cel-go/cel"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+func (t *TestResourceRef) BuildResourceReference() (namespacedName types.NamespacedName, referencedResource *unstructured.Unstructured) {
+	referencedResource = &unstructured.Unstructured{}
+	apiVersionSplit := strings.Split(t.APIVersion, "/")
+	gvk := schema.GroupVersionKind{
+		Version: apiVersionSplit[len(apiVersionSplit)-1],
+		Kind:    t.Kind,
+	}
+	if len(t.APIVersion) > 1 {
+		gvk.Group = apiVersionSplit[0]
+	}
+	referencedResource.SetGroupVersionKind(gvk)
+
+	namespacedName = types.NamespacedName{
+		Namespace: t.Namespace,
+		Name:      t.Name,
+	}
+
+	return
+}
+
+func (t *TestResourceRef) Validate() error {
+	apiVersionSplit := strings.Split(t.APIVersion, "/")
+	switch {
+	case t.APIVersion == "" || (len(apiVersionSplit) != 1 && len(apiVersionSplit) != 2):
+		return fmt.Errorf("apiVersion '%v' not of the format (<group>/)<version>", t.APIVersion)
+	case t.Kind == "":
+		return errors.New("kind not specified")
+	case t.Namespace == "":
+		return errors.New("namespace not specified")
+	case t.Name == "":
+		return errors.New("name not specified")
+	case t.Ref == "":
+		return errors.New("ref not specified")
+	}
+
+	return nil
+}
+
+func (t *TestResourceRef) String() string {
+	return fmt.Sprintf(
+		"apiVersion=%v, kind=%v, namespace=%v, name=%v, ref=%v",
+		t.APIVersion,
+		t.Kind,
+		t.Namespace,
+		t.Name,
+		t.Ref,
+	)
+}
+
+func (t *Assertion) BuildProgram(env *cel.Env) (cel.Program, error) {
+	ast, issues := env.Compile(t.CELExpression)
+	if issues != nil && issues.Err() != nil {
+		return nil, fmt.Errorf("type-check error: %s", issues.Err())
+	}
+
+	prg, err := env.Program(ast)
+	if err != nil {
+		return nil, fmt.Errorf("program construction error: %w", err)
+	}
+
+	return prg, nil
+}

pkg/apis/testharness/v1beta1/test_types.go

@@ -152,6 +152,11 @@ type TestAssert struct {
 	Collectors []*TestCollector `json:"collectors,omitempty"`
 	// Commands is a set of commands to be run as assertions for the current step
 	Commands []TestAssertCommand `json:"commands,omitempty"`
+
+	ResourceRefs []TestResourceRef `json:"resourceRefs,omitempty"`
+
+	AssertAny []*Assertion `json:"assertAny,omitempty"`
+	AssertAll []*Assertion `json:"assertAll,omitempty"`
 }
 
 // TestAssertCommand an assertion based on the result of the execution of a command
@@ -222,6 +227,18 @@ type TestCollector struct {
 	Cmd string `json:"command,omitempty"`
 }
 
+type TestResourceRef struct {
+	APIVersion string `json:"apiVersion,omitempty"`
+	Kind       string `json:"kind,omitempty"`
+	Namespace  string `json:"namespace,omitempty"`
+	Name       string `json:"name,omitempty"`
+	Ref        string `json:"ref,omitempty"`
+}
+
+type Assertion struct {
+	CELExpression string `json:"celExpr,omitempty"`
+}
+
 // DefaultKINDContext defines the default kind context to use.
 const DefaultKINDContext = "kind"
 

pkg/test/case.go

@@ -339,7 +339,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 			continue
 		}
 
-		cl, err = newClient(testStep.Kubeconfig, testStep.Context)(false)
+		cl, err = testutils.NewClient(testStep.Kubeconfig, testStep.Context)(false)
 		if err != nil {
 			setupReport.Failure = report.NewFailure(err.Error(), nil)
 			ts.AddTestcase(setupReport)
@@ -364,7 +364,7 @@ func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
 		tc := report.NewCase("step " + testStep.String())
 		testStep.Client = t.Client
 		if testStep.Kubeconfig != "" {
-			testStep.Client = newClient(testStep.Kubeconfig, testStep.Context)
+			testStep.Client = testutils.NewClient(testStep.Kubeconfig, testStep.Context)
 		}
 		testStep.DiscoveryClient = t.DiscoveryClient
 		if testStep.Kubeconfig != "" {
@@ -532,19 +532,6 @@ func (t *Case) LoadTestSteps() error {
 	return nil
 }
 
-func newClient(kubeconfig, context string) func(bool) (client.Client, error) {
-	return func(bool) (client.Client, error) {
-		config, err := k8s.BuildConfigWithContext(kubeconfig, context)
-		if err != nil {
-			return nil, err
-		}
-
-		return testutils.NewRetryClient(config, client.Options{
-			Scheme: testutils.Scheme(),
-		})
-	}
-}
-
 func newDiscoveryClient(kubeconfig, context string) func() (discovery.DiscoveryInterface, error) {
 	return func() (discovery.DiscoveryInterface, error) {
 		config, err := k8s.BuildConfigWithContext(kubeconfig, context)

pkg/test/step.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/google/cel-go/cel"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -45,6 +46,8 @@ type Step struct {
 	Step   *harness.TestStep
 	Assert *harness.TestAssert
 
+	Programs map[string]cel.Program
+
 	Asserts []client.Object
 	Apply   []client.Object
 	Errors  []client.Object
@@ -412,6 +415,20 @@ func (s *Step) CheckAssertCommands(ctx context.Context, namespace string, comman
 	return testErrors
 }
 
+func (s *Step) CheckAssertExpressions(
+	ctx context.Context,
+	resourceRefs []harness.TestResourceRef,
+	assertAny,
+	assertAll []*harness.Assertion,
+) []error {
+	client, err := s.Client(false)
+	if err != nil {
+		return []error{err}
+	}
+
+	return testutils.RunAssertExpressions(ctx, client, s.Programs, resourceRefs, assertAny, assertAll)
+}
+
 // Check checks if the resources defined in Asserts and Errors are in the correct state.
 func (s *Step) Check(namespace string, timeout int) []error {
 	testErrors := []error{}
@@ -422,6 +439,7 @@ func (s *Step) Check(namespace string, timeout int) []error {
 
 	if s.Assert != nil {
 		testErrors = append(testErrors, s.CheckAssertCommands(context.TODO(), namespace, s.Assert.Commands, timeout)...)
+		testErrors = append(testErrors, s.CheckAssertExpressions(context.TODO(), s.Assert.ResourceRefs, s.Assert.AssertAny, s.Assert.AssertAll)...)
 	}
 
 	for _, expected := range s.Errors {
@@ -533,6 +551,42 @@ func (s *Step) LoadYAML(file string) error {
 			} else {
 				return fmt.Errorf("failed to load TestAssert object from %s: it contains an object of type %T", file, obj)
 			}
+
+			var errs []error
+			for _, resourceRef := range s.Assert.ResourceRefs {
+				if err := resourceRef.Validate(); err != nil {
+					errs = append(errs, fmt.Errorf("validation failed for reference '%v': %w", resourceRef.String(), err))
+				}
+			}
+
+			if len(errs) > 0 {
+				return fmt.Errorf("failed to load TestAssert object from %s: %w", file, errors.Join(errs...))
+			}
+
+			var assertions []*harness.Assertion
+			assertions = append(assertions, s.Assert.AssertAny...)
+			assertions = append(assertions, s.Assert.AssertAll...)
+
+			env, err := testutils.BuildEnv(s.Assert.ResourceRefs)
+			if err != nil {
+				return fmt.Errorf("failed to load TestAssert object from %s: %w", file, err)
+			}
+
+			if len(assertions) > 0 {
+				s.Programs = make(map[string]cel.Program)
+			}
+
+			for _, assertion := range assertions {
+				if prg, err := assertion.BuildProgram(env); err != nil {
+					errs = append(errs, err)
+				} else {
+					s.Programs[assertion.CELExpression] = prg
+				}
+			}
+
+			if len(errs) > 0 {
+				return fmt.Errorf("failed to load TestAssert object from %s: %w", file, errors.Join(errs...))
+			}
 		} else {
 			asserts = append(asserts, obj)
 		}