feat: pre core exemption handling

.github/bundles/exemption/uds-bundle.yaml

@@ -0,0 +1,75 @@
+kind: UDSBundle
+metadata:
+  name: k3d-core-slim-dev
+  description: A UDS bundle for deploying Istio from UDS Core on a development cluster
+  # x-release-please-start-version
+  version: "0.21.0"
+  # x-release-please-end
+
+packages:
+  - name: uds-k3d-dev
+    repository: ghcr.io/defenseunicorns/packages/uds-k3d
+    # renovate: datasource=github-tags depName=defenseunicorns/uds-k3d versioning=semver
+    ref: 0.6.0
+    overrides:
+      uds-dev-stack:
+        minio:
+          variables:
+            - name: buckets
+              description: "Set Minio Buckets"
+              path: buckets
+            - name: svcaccts
+              description: "Minio Service Accounts"
+              path: svcaccts
+            - name: users
+              description: "Minio Users"
+              path: users
+            - name: policies
+              description: "Minio policies"
+              path: policies
+
+  - name: init
+    repository: ghcr.io/defenseunicorns/packages/init
+    # renovate: datasource=github-tags depName=defenseunicorns/zarf versioning=semver
+    ref: v0.33.1
+
+  - name: podinfo
+    repository: ghcr.io/defenseunicorns/uds-cli/podinfo
+    ref: 0.0.1
+    overrides:
+      podinfo:
+        podinfo:
+          values:
+            - path: securityContext
+              value:
+                runAsUser: 0
+                runAsGroup: 0
+
+  - name: core-slim-dev
+    path: ../../../build/
+    # x-release-please-start-version
+    ref: 0.21.0
+    # x-release-please-end
+    overrides:
+      pre-core-exemptions:
+        pre-core-exemptions:
+          values:
+            - path: enabled
+              value: true
+            - path: exemptions
+              value: |
+                - policies:
+                    - DisallowPrivileged
+                    - RequireNonRootUser
+                    - DropAllCapabilities
+                  title: "podinfo1"
+                  matcher:
+                    namespace: podinfo
+                    name: "^podinfo.*"
+                - policies:
+                    - DisallowNodePortServices
+                  title: "podinfo2"
+                  matcher:
+                    namespace: podinfo
+                    name: "^.*-local.*"
+                    kind: service

.github/bundles/uds-bundle.yaml → .github/bundles/infra-bundle/uds-bundle.yaml

@@ -13,7 +13,7 @@ packages:
     ref: v0.33.1
 
   - name: core
-    path: ../../build/
+    path: ../../../build/
     # x-release-please-start-version
     ref: 0.21.0
     # x-release-please-end

.github/filters.yaml

@@ -25,3 +25,5 @@ tempo:
   - "src/tempo/**"
 velero:
   - "src/velero/**"
+pre-core-exemptions:
+  - "src/pre-core-exemptions/**"

.github/workflows/nightly-testing.yaml

@@ -7,7 +7,7 @@ on:
     paths:
       - tasks/iac.yaml
       - .github/workflows/test-eks.yaml
-      - .github/bundles/*
+      - .github/bundles/infra-bundle/*
       - .github/test-infra/buckets-iac/*
 
 jobs:

.github/workflows/publish.yaml

@@ -68,6 +68,11 @@ jobs:
           uds run deploy-standard-bundle --no-progress
           uds run -f tasks/test.yaml validate-packages --no-progress
 
+      # Cluster must be up for generating the CRD
+      - name: Create Exemption CRD Package
+        if: ${{ matrix.flavor == 'upstream' }}
+        run: uds run -f tasks/create.yaml exemption-crd-package
+
       - name: Debug Output
         if: ${{ always() && !inputs.snapshot }}
         uses: ./.github/actions/debug-output
@@ -80,6 +85,10 @@ jobs:
         if: ${{ !inputs.snapshot && matrix.flavor != 'registry1' }}
         run: uds run -f tasks/publish.yaml bundles
 
+      - name: Publish Exemption CRD Package
+        if: ${{ matrix.flavor == 'upstream' }}
+        run: uds run -f tasks/publish.yaml exemption-crd-package
+
       - name: (Snapshot) Publish Standard Package
         if: ${{ inputs.snapshot }}
         run: uds run -f tasks/publish.yaml standard-package --set FLAVOR=${{ matrix.flavor }} --set TARGET_REPO="ghcr.io/defenseunicorns/packages/uds/snapshots" --set VERSION="${SNAPSHOT_VERSION}"

.github/workflows/test-eks.yaml

@@ -52,7 +52,7 @@ jobs:
         run: ZARF_ARCHITECTURE=amd64 uds run -f tasks/create.yaml standard-package
 
       - name: Create Core Bundle
-        run: uds create .github/bundles --confirm
+        run: uds create .github/bundles/infra-bundle --confirm
 
       - name: Create Cluster
         run: uds run -f tasks/iac.yaml create-cluster
@@ -64,13 +64,13 @@ jobs:
 
       - name: Deploy Core Bundle
         env:
-          UDS_CONFIG: .github/bundles/uds-config.yaml
-        run: uds deploy .github/bundles/uds-bundle-uds-core-eks-nightly-*.tar.zst --confirm
+          UDS_CONFIG: .github/bundles/infra-bundle/uds-config.yaml
+        run: uds deploy .github/bundles/infra-bundle/uds-bundle-uds-core-eks-nightly-*.tar.zst --confirm
         timeout-minutes: 20
 
       - name: Remove UDS Core
         if: always()
-        run: uds remove .github/bundles/uds-bundle-uds-core-eks-*.tar.zst --confirm
+        run: uds remove .github/bundles/infra-bundle/uds-bundle-uds-core-eks-*.tar.zst --confirm
         timeout-minutes: 10
         continue-on-error: true
 

.github/workflows/test.yaml

@@ -57,9 +57,13 @@ jobs:
         run: uds run registry-login --set REGISTRY=registry1.dso.mil --set REGISTRY_USERNAME=${{ secrets.IRON_BANK_ROBOT_USERNAME }} --set REGISTRY_PASSWORD=${{ secrets.IRON_BANK_ROBOT_PASSWORD }} --set REGISTRY_RETRY_INTERVAL=90
 
       - name: Test a single source package
-        if: ${{ inputs.package != 'all' && inputs.test_type == 'install' }}
+        if: ${{ inputs.package != 'all' && inputs.package != 'pre-core-exemptions' && inputs.test_type == 'install' }}
         run: uds run test-single-package --set FLAVOR=${{ inputs.flavor }}
 
+      - name: Test pre core exemptions package
+        if: ${{ inputs.package == 'pre-core-exemptions' && inputs.test_type == 'install'}}
+        run: uds run test-pre-core-exemptions
+
       - name: Test UDS Core Install
         if: ${{ inputs.package == 'all' && inputs.test_type == 'install' }}
         run: uds run test-uds-core --set FLAVOR=${{ inputs.flavor }}

docs/CONFIGURE_POLICY_EXEMPTIONS.md

@@ -24,5 +24,38 @@ shared:
 
 variables:
  # package specific variables here
-
-```
+```
+
+## Handling Pre Core Exemptions
+
+If you find that you are deploying a resource before uds-core that requires an exemption CR once uds-core is deployed there are two options:
+
+1) Pass exemptions to the core `pre-core-exemptions` package as helm values overrides. See [Pre Core Exemptions Docs](../src/pre-core-exemptions/README.md)
+
+2) Deploy the exemption CRD as a package in your bundle before your pre-core resource.
+   (e.g.)
+   ```yaml
+   kind: UDSBundle
+   metadata:
+     name: example CRD
+
+   packages:
+     - name: exemption-crd
+       repository: ghcr.io/defenseunicorns/packages/uds/exemption-crd
+       ref: <same-ref-as-core>
+
+     # Where the exemption CR lives
+     - name: custom-init
+       repository: ghcr.io/custom-init
+       ref: v0.1.0
+
+     - name: core
+       path: ghcr.io/defenseunicorns/packages/uds/core
+       ref: 0.20.0-upstream
+   ```
+
+> Warning 
+>
+> Though both options have been tested, there is no guarantee
+> that upgrades, specifically to the CRD package and your CR,
+> will not break your deployment.

packages/exemption-crd/zarf.yaml

@@ -0,0 +1,16 @@
+kind: ZarfPackageConfig
+metadata:
+  name: uds-core-exemption-crd
+  yolo: true
+  # x-release-please-start-version
+  version: 0.20.0
+  # x-release-please-end
+
+components:
+  - name: exemption-crd
+    required: true
+    manifests:
+      - name: crd
+        files:
+        # generated in CI
+          - ./crd.yaml

packages/slim-dev/zarf.yaml

@@ -35,6 +35,11 @@ components:
     import:
       path: ../standard
 
+  - name: pre-core-exemptions
+    required: true
+    import:
+      path: ../standard
+
   # Keycloak
   - name: keycloak
     required: true

packages/standard/zarf.yaml

@@ -42,6 +42,11 @@ components:
       path: ../../dist
       name: module
 
+  - name: pre-core-exemptions
+    required: true
+    import:
+      path: ../../src/pre-core-exemptions
+
   # Metrics Server
   - name: metrics-server
     required: true

release-please-config.json

@@ -11,7 +11,7 @@
       ],
       "versioning": "default",
       "extra-files": [
-        ".github/bundles/uds-bundle.yaml",
+        ".github/bundles/infra-bundle/uds-bundle.yaml",
         "README.md",
         "packages/slim-dev/zarf.yaml",
         "packages/standard/zarf.yaml",

src/pepr/tasks.yaml

@@ -10,6 +10,8 @@ tasks:
         env:
           - "PEPR_WATCH_MODE=true"
 
+      - cmd: "kubectl get crd exemptions.uds.dev -o yaml > packages/exemption-crd/crd.yaml"
+
       - cmd: "npx kubernetes-fluent-client crd packages.uds.dev src/pepr/operator/crd/generated"
 
       - cmd: "npx kubernetes-fluent-client crd exemptions.uds.dev src/pepr/operator/crd/generated"

src/pre-core-exemptions/README.md

@@ -0,0 +1,53 @@
+# Pre Core Exemptions
+
+This package serves as a way for users deploying uds-core to pass exemptions to core for things that are deployed before core.
+
+For example, if a team is deploying a bundle containing a custom init package that has rook-ceph, the first time it
+deploys everything will be fine, but if it cycles for whatever reason once core is deployed then rook-ceph will be denied by Pepr policies. Thus an exemption CR is needed.
+The problem, though, is the init package can't deploy an exemption resource when the exemption CRD has not yet been deployed by core. 
+
+This package solves that timing issue by applying whatever exemptions are given to it as soon as the Pepr core module has been successfully deployed.
+
+## How to Use
+
+Add helm values overrides to your `uds-bundle.yaml`:
+
+```yaml
+kind: UDSBundle
+metadata:
+  name: example helm overrides
+
+packages:
+  - name: custom-init
+    repository: ghcr.io/custom-init
+    ref: v0.1.0
+
+  - name: core
+    path: ghcr.io/defenseunicorns/packages/uds/core
+    ref: 0.20.0-upstream
+    overrides:
+      pre-core-exemptions:
+        pre-core-exemptions:
+          values:
+            - path: enabled
+              value: true
+            - path: exemptions
+              value: |
+                - policies:
+                    - DisallowPrivileged
+                    - RequireNonRootUser
+                    - DropAllCapabilities
+                  title: "podinfo1"
+                  matcher:
+                    namespace: podinfo
+                    name: "^podinfo.*"
+                    test: 1
+                - policies:
+                    - DisallowNodePortServices
+                  title: "podinfo2"
+                  matcher:
+                    namespace: podinfo
+                    name: "^.*-local.*"
+                    kind: service
+                    test: 2
+```

src/pre-core-exemptions/chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

src/pre-core-exemptions/chart/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: pre-core-exemptions
+description: Helm chart for exemptions that are required before core but couldn't be applied before core
+type: application
+version: 0.1.0

src/pre-core-exemptions/chart/templates/uds-exemption.yaml

@@ -0,0 +1,9 @@
+{{- if .Values.enabled }}
+apiVersion: uds.dev/v1alpha1
+kind: Exemption
+metadata:
+  name: pre-core-exemptions
+  namespace: {{ .Values.namespace }}
+spec:
+  exemptions: {{ .Values.exemptions | nindent 4 }}
+{{- end }}

src/pre-core-exemptions/chart/values.yaml

@@ -0,0 +1,3 @@
+enabled: false
+namespace: uds-policy-exemptions
+exemptions: {}

src/pre-core-exemptions/tasks.yaml

@@ -0,0 +1,29 @@
+# works in conjunction with .github/bundles/exemption/uds-bundle.yaml
+variables:
+  - name: PODINFO_NAME
+
+tasks:
+  - name: validate
+    actions:
+      - description: Validate exemptions are applied
+        wait:
+          cluster:
+            kind: exemption
+            name: pre-core-exemptions
+            namespace: uds-policy-exemptions
+            condition: exists
+
+      - description: Get podinfo name
+        cmd: echo $(uds zarf tools kubectl get pods -n podinfo -o jsonpath='{.items[0].metadata.name}')
+        setVariables:
+          - name: PODINFO_NAME
+
+      - description: Cycle podinfo
+        cmd: uds zarf tools kubectl delete pod -n podinfo ${PODINFO_NAME}
+
+      - description: Check for podinfo to come back up
+        wait:
+          cluster:
+            kind: Pod
+            name: $PODINFO_NAME
+            namespace: podinfo