feat: add test scenarios support (cherry-pick #1576)

.crds/chainsaw.kyverno.io_tests.yaml

@@ -780,6 +780,31 @@ spec:
                   namespace.
                 type: object
                 x-kubernetes-preserve-unknown-fields: true
+              scenarios:
+                description: Scenarios defines test scenarios.
+                items:
+                  description: Scenario defines per scenario bindings.
+                  properties:
+                    bindings:
+                      description: Bindings defines binding key/values.
+                      items:
+                        description: Binding represents a key/value set as a binding
+                          in an executing test.
+                        properties:
+                          name:
+                            description: Name the name of the binding.
+                            pattern: ^(?:\w+|\(.+\))$
+                            type: string
+                          value:
+                            description: Value value of the binding.
+                            x-kubernetes-preserve-unknown-fields: true
+                        required:
+                        - name
+                        - value
+                        type: object
+                      type: array
+                  type: object
+                type: array
               skip:
                 description: Skip determines whether the test should skipped.
                 type: boolean

.schemas/json/test-chainsaw-v1alpha1.json

@@ -1464,6 +1464,51 @@
           "description": "NamespaceTemplate defines a template to create the test namespace.",
           "x-kubernetes-preserve-unknown-fields": true
         },
+        "scenarios": {
+          "description": "Scenarios defines test scenarios.",
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "description": "Scenario defines per scenario bindings.",
+            "type": [
+              "object",
+              "null"
+            ],
+            "properties": {
+              "bindings": {
+                "description": "Bindings defines binding key/values.",
+                "type": [
+                  "array",
+                  "null"
+                ],
+                "items": {
+                  "description": "Binding represents a key/value set as a binding in an executing test.",
+                  "type": [
+                    "object",
+                    "null"
+                  ],
+                  "required": [
+                    "name",
+                    "value"
+                  ],
+                  "properties": {
+                    "name": {
+                      "description": "Name the name of the binding.",
+                      "type": "string",
+                      "pattern": "^(?:\\w+|\\(.+\\))$"
+                    },
+                    "value": {
+                      "description": "Value value of the binding.",
+                      "x-kubernetes-preserve-unknown-fields": true
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "skip": {
           "description": "Skip determines whether the test should skipped.",
           "type": [

pkg/apis/v1alpha1/test.go

@@ -64,6 +64,10 @@ type TestSpec struct {
 	// +optional
 	NamespaceTemplate *Any `json:"namespaceTemplate,omitempty"`
 
+	// Scenarios defines test scenarios.
+	// +optional
+	Scenarios []Scenario `json:"scenarios,omitempty"`
+
 	// Bindings defines additional binding key/values.
 	// +optional
 	Bindings []Binding `json:"bindings,omitempty"`
@@ -91,3 +95,10 @@ type TestSpec struct {
 	// +kubebuilder:validation:Enum:=Orphan;Background;Foreground
 	DeletionPropagationPolicy *metav1.DeletionPropagation `json:"deletionPropagationPolicy,omitempty"`
 }
+
+// Scenario defines per scenario bindings.
+type Scenario struct {
+	// Bindings defines binding key/values.
+	// +optional
+	Bindings []Binding `json:"bindings,omitempty"`
+}

pkg/data/crds/chainsaw.kyverno.io_tests.yaml

@@ -780,6 +780,31 @@ spec:
                   namespace.
                 type: object
                 x-kubernetes-preserve-unknown-fields: true
+              scenarios:
+                description: Scenarios defines test scenarios.
+                items:
+                  description: Scenario defines per scenario bindings.
+                  properties:
+                    bindings:
+                      description: Bindings defines binding key/values.
+                      items:
+                        description: Binding represents a key/value set as a binding
+                          in an executing test.
+                        properties:
+                          name:
+                            description: Name the name of the binding.
+                            pattern: ^(?:\w+|\(.+\))$
+                            type: string
+                          value:
+                            description: Value value of the binding.
+                            x-kubernetes-preserve-unknown-fields: true
+                        required:
+                        - name
+                        - value
+                        type: object
+                      type: array
+                  type: object
+                type: array
               skip:
                 description: Skip determines whether the test should skipped.
                 type: boolean

pkg/data/schemas/json/test-chainsaw-v1alpha1.json

@@ -1464,6 +1464,51 @@
           "description": "NamespaceTemplate defines a template to create the test namespace.",
           "x-kubernetes-preserve-unknown-fields": true
         },
+        "scenarios": {
+          "description": "Scenarios defines test scenarios.",
+          "type": [
+            "array",
+            "null"
+          ],
+          "items": {
+            "description": "Scenario defines per scenario bindings.",
+            "type": [
+              "object",
+              "null"
+            ],
+            "properties": {
+              "bindings": {
+                "description": "Bindings defines binding key/values.",
+                "type": [
+                  "array",
+                  "null"
+                ],
+                "items": {
+                  "description": "Binding represents a key/value set as a binding in an executing test.",
+                  "type": [
+                    "object",
+                    "null"
+                  ],
+                  "required": [
+                    "name",
+                    "value"
+                  ],
+                  "properties": {
+                    "name": {
+                      "description": "Name the name of the binding.",
+                      "type": "string",
+                      "pattern": "^(?:\\w+|\\(.+\\))$"
+                    },
+                    "value": {
+                      "description": "Value value of the binding.",
+                      "x-kubernetes-preserve-unknown-fields": true
+                    }
+                  }
+                }
+              }
+            }
+          }
+        },
         "skip": {
           "description": "Skip determines whether the test should skipped.",
           "type": [

pkg/runner/processors/info.go

@@ -5,8 +5,9 @@ import (
 )
 
 type TestInfo struct {
-	Id       int
-	Metadata metav1.ObjectMeta
+	Id         int
+	ScenarioId int
+	Metadata   metav1.ObjectMeta
 }
 
 type StepInfo struct {

pkg/runner/processors/tests.go

@@ -137,24 +137,45 @@
 			logging.Log(ctx, logging.Internal, logging.ErrorStatus, color.BoldRed, logging.ErrSection(err))
 			failer.FailNow(ctx)
 		}
-		t.Run(name, func(t *testing.T) {
-			t.Helper()
-			t.Cleanup(func() {
-				if t.Failed() {
-					p.shouldFailFast.Store(true)
+		var scenarios []discovery.Test
+		if test.Test != nil {
+			if len(test.Spec.Scenarios) == 0 {
+				scenarios = append(scenarios, test)
+			} else {
+				for s := range test.Spec.Scenarios {
+					scenario := test.Spec.Scenarios[s]
+					test := test
+					test.Test = test.Test.DeepCopy()
+					test.Spec.Scenarios = nil
+					bindings := scenario.Bindings
+					bindings = append(bindings, test.Spec.Bindings...)
+					test.Spec.Bindings = bindings
+					scenarios = append(scenarios, test)
 				}
-			})
-			processor := p.CreateTestProcessor(test)
-			info := TestInfo{
-				Id:       i + 1,
-				Metadata: test.ObjectMeta,
 			}
-			processor.Run(
-				testing.IntoContext(ctx, t),
-				apibindings.RegisterNamedBinding(ctx, bindings, "test", info),
-				nspacer,
-			)
-		})
+		}
+		for s := range scenarios {
+			test := scenarios[s]
+			t.Run(name, func(t *testing.T) {
+				t.Helper()
+				t.Cleanup(func() {
+					if t.Failed() {
+						p.shouldFailFast.Store(true)
+					}
+				})
+				processor := p.CreateTestProcessor(test)
+				info := TestInfo{
+					Id:         i + 1,
+					ScenarioId: s + 1,
+					Metadata:   test.ObjectMeta,
+				}
+				processor.Run(
+					testing.IntoContext(ctx, t),
+					apibindings.RegisterNamedBinding(ctx, bindings, "test", info),
+					nspacer,
+				)
+			})
+		}
 	}
 }
 

testdata/e2e/examples/CATALOG.md

@@ -25,6 +25,7 @@
 - [outputs](outputs/README.md)
 - [patch](patch/README.md)
 - [quick-start](quick-start/README.md)
+- [scenarios](scenarios/README.md)
 - [script-env](script-env/README.md)
 - [sleep](sleep/README.md)
 - [template](template/README.md)

testdata/e2e/examples/scenarios/README.md

@@ -0,0 +1,22 @@
+# Test: `scenarios`
+
+*No description*
+
+## Steps
+
+| # | Name | Bindings | Try | Catch | Finally | Cleanup |
+|:-:|---|:-:|:-:|:-:|:-:|
+| 1 | [step-1](#step-step-1) | 0 | 1 | 0 | 0 | 0 |
+
+### Step: `step-1`
+
+*No description*
+
+#### Try
+
+| # | Operation | Bindings | Outputs | Description |
+|:-:|---|:-:|:-:|---|
+| 1 | `script` | 0 | 0 | *No description* |
+
+---
+

testdata/e2e/examples/scenarios/chainsaw-test.yaml

@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: scenarios
+spec:
+  scenarios:
+  - bindings:
+    - name: message
+      value: hello
+  - bindings:
+    - name: message
+      value: goodbye
+  steps:
+  - try:
+    - script:
+        env:
+        - name: message
+          value: ($message)
+        content: echo $message
+        check:
+         ($stdout): ((to_number($test.scenarioId) == `1` && 'hello') || 'goodbye')

website/docs/reference/apis/chainsaw.v1alpha1.md

@@ -316,6 +316,7 @@ during the testing process.</p>
 - [ActionBindings](#chainsaw-kyverno-io-v1alpha1-ActionBindings)
 - [ActionEnv](#chainsaw-kyverno-io-v1alpha1-ActionEnv)
 - [Output](#chainsaw-kyverno-io-v1alpha1-Output)
+- [Scenario](#chainsaw-kyverno-io-v1alpha1-Scenario)
 - [TestSpec](#chainsaw-kyverno-io-v1alpha1-TestSpec)
 - [TestStepSpec](#chainsaw-kyverno-io-v1alpha1-TestStepSpec)
 
@@ -716,6 +717,19 @@ If a resource doesn't exist yet in the cluster it will fail.</p>
 
 - [ConfigurationSpec](#chainsaw-kyverno-io-v1alpha1-ConfigurationSpec)
 
+## Scenario     {#chainsaw-kyverno-io-v1alpha1-Scenario}
+
+**Appears in:**
+
+- [TestSpec](#chainsaw-kyverno-io-v1alpha1-TestSpec)
+
+<p>Scenario defines per scenario bindings.</p>
+
+
+| Field | Type | Required | Inline | Description |
+|---|---|---|---|---|
+| `bindings` | [`[]Binding`](#chainsaw-kyverno-io-v1alpha1-Binding) |  |  | <p>Bindings defines binding key/values.</p> |
+
 ## Script     {#chainsaw-kyverno-io-v1alpha1-Script}
 
 **Appears in:**
@@ -771,6 +785,7 @@ If a resource doesn't exist yet in the cluster it will fail.</p>
 | `template` | `bool` |  |  | <p>Template determines whether resources should be considered for templating.</p> |
 | `namespace` | `string` |  |  | <p>Namespace determines whether the test should run in a random ephemeral namespace or not.</p> |
 | `namespaceTemplate` | `policy/v1alpha1.Any` |  |  | <p>NamespaceTemplate defines a template to create the test namespace.</p> |
+| `scenarios` | [`[]Scenario`](#chainsaw-kyverno-io-v1alpha1-Scenario) |  |  | <p>Scenarios defines test scenarios.</p> |
 | `bindings` | [`[]Binding`](#chainsaw-kyverno-io-v1alpha1-Binding) |  |  | <p>Bindings defines additional binding key/values.</p> |
 | `steps` | [`[]TestStep`](#chainsaw-kyverno-io-v1alpha1-TestStep) | :white_check_mark: |  | <p>Steps defining the test.</p> |
 | `catch` | [`[]CatchFinally`](#chainsaw-kyverno-io-v1alpha1-CatchFinally) |  |  | <p>Catch defines what the steps will execute when an error happens. This will be combined with catch handlers defined at the step level.</p> |