feat: test VAPs generated by pod security cel policies

.github/actions/install-crds/action.yaml

@@ -0,0 +1,10 @@
+name: "Install CRDs"
+description: "Installs all CRDs for chainsaw tests"
+runs:
+  using: "composite"
+  steps:
+    - name: Install CRDs
+      shell: bash
+      run: |
+        set -e
+        kubectl apply -f ./.chainsaw/crds

.github/actions/run-tests/action.yaml

@@ -1,9 +1,15 @@
 name: "Runs E2E Tests"
 description: "Runs E2E tests using chainsaw"
 inputs:
+  k8s-version:
+    description: "Kubernetes version"
+    required: false
   tests:
     description: "Test regex"
     required: true
+  test-file:
+    description: "Name of the chainsaw test file"
+    default: chainsaw-test
 runs:
   using: "composite"
   steps:
@@ -13,4 +19,14 @@ runs:
       shell: bash
       run: |
         set -e
-        chainsaw test --config .chainsaw.yaml --include-test-regex '^chainsaw$/${{ inputs.tests }}' --no-color=false
+
+        K8S_VERSION="${{ inputs.k8s-version }}"
+        TEST_FILE="${{ inputs.test-file }}"
+        TESTS="${{ inputs.tests }}"
+
+        if [[ "$TEST_FILE" == "chainsaw-test-vap" && -n "$K8S_VERSION" ]] && 
+          [[ "$K8S_VERSION" == "v1.26."* || "$K8S_VERSION" == "v1.27."* ]]; then
+          chainsaw test --config .chainsaw.yaml --include-test-regex '^chainsaw$/${{ inputs.tests }}' --test-file='${{ inputs.test-file }}' --no-color=false --selector='!skipForVapAlpha'
+        else
+          chainsaw test --config .chainsaw.yaml --include-test-regex '^chainsaw$/${{ inputs.tests }}' --test-file='${{ inputs.test-file }}' --no-color=false
+        fi

.github/actions/setup-env/action.yaml

@@ -40,12 +40,6 @@ runs:
         set -e
         kubectl create -f https://github.com/kyverno/kyverno/raw/main/config/install-latest-testing.yaml
     - name: Wait for kyverno ready
-      shell: bash
-      run: |
-        set -e
-        kubectl wait --namespace kyverno --for=condition=ready pod --selector '!job-name' --timeout=60s
+      uses: ./.github/actions/wait-for-kyverno-ready
     - name: Install CRDs
-      shell: bash
-      run: |
-        set -e
-        kubectl apply -f ./.chainsaw/crds
+      uses: ./.github/actions/install-crds

.github/actions/wait-for-kyverno-ready/action.yaml

@@ -0,0 +1,10 @@
+name: "Wait for Kyverno To Become Ready"
+description: "Waits for Kyverno to become ready before running chainsaw tests"
+runs:
+  using: "composite"
+  steps:
+    - name: Wait for kyverno ready
+      shell: bash
+      run: |
+        set -e
+        kubectl wait --namespace kyverno --for=condition=ready pod --selector '!job-name' --timeout=60s

.github/scripts/config/generate-validating-admission-policy/generate-vap-cr.yaml

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: admission-controller
+    app.kubernetes.io/instance: kyverno
+    app.kubernetes.io/part-of: kyverno
+  name: kyverno:generate-validatingadmissionpolicy
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingadmissionpolicies
+  - validatingadmissionpolicybindings
+  verbs:
+  - create
+  - update
+  - delete
+  - list

.github/scripts/config/generate-validating-admission-policy/install-kyverno.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+echo -e "\nDownloading the latest Kyverno installation YAML file..."
+wget -O install-latest-testing.yaml https://github.com/kyverno/kyverno/raw/main/config/install-latest-testing.yaml
+
+echo -e "\nEnabling Validating Admission Policy generation..."
+sed -i 's/--generateValidatingAdmissionPolicy=false/--generateValidatingAdmissionPolicy=true/g' ./install-latest-testing.yaml
+
+echo -e "\nInstalling Kyverno in the cluster..."
+kubectl create -f ./install-latest-testing.yaml
+
+echo -e "\nGranting permissions to Kyverno for VAP generation..."
+kubectl create -f ./.github/scripts/config/generate-validating-admission-policy/generate-vap-cr.yaml

.github/scripts/config/kind/vap-v1alpha1.yaml

@@ -0,0 +1,11 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+featureGates:
+  ValidatingAdmissionPolicy: true
+runtimeConfig:
+  admissionregistration.k8s.io/v1alpha1: true
+nodes:
+  - role: control-plane
+  - role: worker
+  - role: worker
+  - role: worker

.github/scripts/config/kind/vap-v1beta1.yaml

@@ -0,0 +1,12 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+featureGates:
+  ValidatingAdmissionPolicy: true
+runtimeConfig:
+  admissionregistration.k8s.io/v1beta1: true
+  admissionregistration.k8s.io/v1alpha1: true
+nodes:
+  - role: control-plane
+  - role: worker
+  - role: worker
+  - role: worker

.github/workflows/cel-test.yml

@@ -63,3 +63,73 @@ jobs:
         uses: ./.github/actions/run-tests
         with:
           tests: ${{ matrix.tests }}
+
+  validatingadmissionpolicies-v1alpha1:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version:
+          - name: v1.27
+            version: v1.27.13
+        tests:
+          - ^pod-security-cel$
+    name: ${{ matrix.k8s-version.name }} - validating-admission-policies - ${{ matrix.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Create kind cluster
+        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+        with:
+          node_image: kindest/node:${{ matrix.k8s-version.version }}
+          cluster_name: kind
+          config: ./.github/scripts/config/kind/vap-v1alpha1.yaml
+      - name: Install latest kyverno with VAP generation enabled
+        run: ./.github/scripts/config/generate-validating-admission-policy/install-kyverno.sh
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/wait-for-kyverno-ready
+      - name: Install CRDs
+        uses: ./.github/actions/install-crds
+      - name: Run VAP tests
+        uses: ./.github/actions/run-tests
+        with: 
+          tests: ${{ matrix.tests }}
+          test-file: chainsaw-test-vap
+          k8s-version: ${{ matrix.k8s-version.version }}
+
+  validatingadmissionpolicies-tests-above-1-28:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        k8s-version:
+          - name: v1.28
+            version: v1.28.9
+          - name: v1.29
+            version: v1.29.4
+          - name: v1.30
+            version: v1.30.0
+        tests:
+          - ^pod-security-cel$
+    name: ${{ matrix.k8s-version.name }} - validating-admission-policies - ${{ matrix.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Create kind cluster
+        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
+        with:
+          node_image: kindest/node:${{ matrix.k8s-version.version }}
+          cluster_name: kind
+          config: ./.github/scripts/config/kind/vap-v1beta1.yaml
+      - name: Install latest kyverno with VAP generation enabled
+        run: ./.github/scripts/config/generate-validating-admission-policy/install-kyverno.sh
+      - name: Wait for kyverno ready
+        uses: ./.github/actions/wait-for-kyverno-ready
+      - name: Install CRDs
+        uses: ./.github/actions/install-crds
+      - name: Run VAP tests
+        uses: ./.github/actions/run-tests
+        with: 
+          tests: ${{ matrix.tests }}
+          test-file: chainsaw-test-vap
+          k8s-version: ${{ matrix.k8s-version.version }}

pod-security-cel/baseline/disallow-capabilities/.chainsaw-test/chainsaw-test-vap.yaml

@@ -0,0 +1,37 @@
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: disallow-capabilities
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: ../disallow-capabilities.yaml
+    - patch:
+        resource:
+          apiVersion: kyverno.io/v1
+          kind: ClusterPolicy
+          metadata:
+            name: disallow-capabilities
+          spec:
+            validationFailureAction: Enforce
+    - assert:
+        file: policy-ready.yaml
+  - name: step-02
+    try:
+    - assert:
+        file: vap-assert.yaml
+    - assert:
+        file: vapbinding-assert.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: pod-good.yaml
+    - apply:
+        expect:
+        - check:
+            ($error != null): true
+        file: pod-bad.yaml
+

pod-security-cel/baseline/disallow-capabilities/.chainsaw-test/vap-assert.yaml

@@ -0,0 +1,12 @@
+apiVersion: admissionregistration.k8s.io/v1alpha1
+kind: ValidatingAdmissionPolicy
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kyverno
+  name: disallow-capabilities
+  ownerReferences:
+  - apiVersion: kyverno.io/v1
+    kind: ClusterPolicy
+    name: disallow-capabilities
+spec:
+  failurePolicy: Fail

pod-security-cel/baseline/disallow-capabilities/.chainsaw-test/vapbinding-assert.yaml

@@ -0,0 +1,13 @@
+apiVersion: admissionregistration.k8s.io/v1alpha1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kyverno
+  name: disallow-capabilities-binding
+  ownerReferences:
+  - apiVersion: kyverno.io/v1
+    kind: ClusterPolicy
+    name: disallow-capabilities
+spec:
+  policyName: disallow-capabilities
+  validationActions: [Deny]

pod-security-cel/baseline/disallow-host-namespaces/.chainsaw-test/chainsaw-test-vap.yaml

@@ -0,0 +1,39 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: disallow-host-namespaces
+spec:
+  # disable templating because it can cause issues with CEL expressions
+  template: false
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: ../disallow-host-namespaces.yaml
+    - patch:
+        resource:
+          apiVersion: kyverno.io/v1
+          kind: ClusterPolicy
+          metadata:
+            name: disallow-host-namespaces
+          spec:
+            validationFailureAction: Enforce
+    - assert:
+        file: policy-ready.yaml
+  - name: step-02
+    try:
+    - assert:
+        file: vap-assert.yaml
+    - assert:
+        file: vapbinding-assert.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: pod-good.yaml
+    - apply:
+        expect:
+        - check:
+            ($error != null): true
+        file: pod-bad.yaml

pod-security-cel/baseline/disallow-host-namespaces/.chainsaw-test/vap-assert.yaml

@@ -0,0 +1,12 @@
+apiVersion: admissionregistration.k8s.io/v1alpha1
+kind: ValidatingAdmissionPolicy
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kyverno
+  name: disallow-host-namespaces
+  ownerReferences:
+  - apiVersion: kyverno.io/v1
+    kind: ClusterPolicy
+    name: disallow-host-namespaces
+spec:
+  failurePolicy: Fail

pod-security-cel/baseline/disallow-host-namespaces/.chainsaw-test/vapbinding-assert.yaml

@@ -0,0 +1,13 @@
+apiVersion: admissionregistration.k8s.io/v1alpha1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kyverno
+  name: disallow-host-namespaces-binding
+  ownerReferences:
+  - apiVersion: kyverno.io/v1
+    kind: ClusterPolicy
+    name: disallow-host-namespaces
+spec:
+  policyName: disallow-host-namespaces
+  validationActions: [Deny]

pod-security-cel/baseline/disallow-host-path/.chainsaw-test/chainsaw-test-vap.yaml

@@ -0,0 +1,37 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: disallow-host-path
+spec:
+  steps:
+  - name: step-01
+    try:
+    - apply:
+        file: ../disallow-host-path.yaml
+    - patch:
+        resource:
+          apiVersion: kyverno.io/v1
+          kind: ClusterPolicy
+          metadata:
+            name: disallow-host-path
+          spec:
+            validationFailureAction: Enforce
+    - assert:
+        file: policy-ready.yaml
+  - name: step-02
+    try:
+    - assert:
+        file: vap-assert.yaml
+    - assert:
+        file: vapbinding-assert.yaml
+  - name: step-03
+    try:
+    - apply:
+        file: pod-good.yaml
+    - apply:
+        expect:
+        - check:
+            ($error != null): true
+        file: pod-bad.yaml

pod-security-cel/baseline/disallow-host-path/.chainsaw-test/vap-assert.yaml

@@ -0,0 +1,12 @@
+apiVersion: admissionregistration.k8s.io/v1alpha1
+kind: ValidatingAdmissionPolicy
+metadata:
+  labels:
+    app.kubernetes.io/managed-by: kyverno
+  name: disallow-host-path
+  ownerReferences:
+  - apiVersion: kyverno.io/v1
+    kind: ClusterPolicy
+    name: disallow-host-path
+spec:
+  failurePolicy: Fail