Reduce disk pressure in workflows (#1847)

## Description

This PR reduces disk pressure further in the release and upgrade
workflows and refactors the upgrade workflow to use ENV_VARs instead of
manually editing the zarf-config.toml.

## Related Issue

Fixes #N/A

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [X] Other (security config, docs update, etc)

## Checklist before merging

- [X] Test, docs, adr added or updated as needed
- [X] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed

.github/workflows/release.yml

@@ -70,19 +70,22 @@ jobs:
         if: always()
         uses: ./.github/actions/save-logs
 
+      # Builds init packages since GoReleaser won't handle this for us
+      - name: Create release time CVE report
+        run: "make cve-report"
+
+      # Before we run GoReleaser we need to aggressively cleanup files to reduce disk pressure
       - name: Cleanup files
         run: |
-          sudo rm -rf zarf-sbom /tmp/zarf-*
+          lsblk -f
+          sudo rm -rf zarf-sbom /tmp/zarf-* src/ui/node_modules
           sudo env "PATH=$PATH" CI=true make delete-packages
           sudo build/zarf tools clear-cache
-          sudo docker system prune --all
+          sudo docker system prune --all --force
+          sudo k3s crictl rmi --prune
           go clean -cache
           lsblk -f
 
-      # Builds init packages since GoReleaser won't handle this for us
-      - name: Create release time CVE report
-        run: "make cve-report"
-
       # Set up AWS credentials for GoReleaser to upload backups of artifacts to S3
       - name: Set AWS Credentials
         uses: aws-actions/configure-aws-credentials@5727f247b64f324ec403ac56ae05e220fd02b65f # v2-node16

.github/workflows/test-upgrade.yml

@@ -23,6 +23,9 @@ concurrency:
 
 jobs:
   validate:
+    env:
+      # Reduce the kubelet eviction minimums reduce the chance for disk pressure causing evictions during the tests
+      ZARF_PACKAGE_DEPLOY_SET_K3S_ARGS: --disable traefik --kubelet-arg "eviction-hard=imagefs.available<1%,nodefs.available<1%" --kubelet-arg "eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%"
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
@@ -39,14 +42,6 @@ jobs:
         with:
           download-init-package: true
 
-      # Note we move the kubelet data to reduce the chance for disk pressure during the tests
-      - name: Move the kubelet root directory for 'k3s'
-        run: |
-          sudo mkdir -p /mnt/kubelet-data
-          echo >> zarf-config.toml
-          echo [package.deploy.set] >> zarf-config.toml
-          echo "k3s_args = '--disable traefik --kubelet-arg \"eviction-hard=imagefs.available<1%,nodefs.available<1%\" --kubelet-arg \"eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%\"'" >> zarf-config.toml
-
       - name: Initialize the cluster with the release version
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
         #       in a previous step. This test run will the current release to create a K3s cluster.
@@ -80,18 +75,19 @@ jobs:
       - name: "Cleanup after running tests"
         # NOTE: This reduces disk pressure before the upgrade-specific tests begin
         run: |
-          sudo lsblk -f
+          lsblk -f
           sudo k3s crictl images
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade
 
-          sudo k3s crictl rmi --prune
-          sudo rm -rf zarf-sbom /tmp/zarf-*/
+          sudo rm -rf zarf-sbom /tmp/zarf-*/ src/ui/node_modules
           sudo env "PATH=$PATH" CI=true make delete-packages
           sudo build/zarf tools clear-cache
+          sudo docker system prune --all --force
+          sudo k3s crictl rmi --prune
 
-          sudo lsblk -f
+          lsblk -f
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
 
       - name: Run the upgrade tests

.goreleaser.yaml

@@ -23,11 +23,7 @@ builds:
 # Save the built artifacts as binaries (instead of wrapping them in a tarball)
 archives:
   - format: binary
-    name_template: "{{ .ProjectName }}_{{ .Tag }}_{{ .Os }}_{{ .Arch }}"
-    replacements:
-      darwin: Darwin
-      linux: Linux
-      windows: Windows
+    name_template: "{{ .ProjectName }}_{{ .Tag }}_{{- title .Os }}_{{ .Arch }}"
 
 # generate a sha256 checksum of all release artifacts
 # NOTE: We are explicitly adding the init-packages that are built prior to GoReleaser stage in the GitHub Actions workflow
@@ -40,6 +36,8 @@ checksum:
 # generate sboms for each binary artifact
 sboms:
   - artifacts: binary
+    documents:
+      - "sbom_{{ .ProjectName }}_{{ .Tag }}_{{- title .Os }}_{{ .Arch }}.sbom"
 
 snapshot:
   name_template: "{{ incpatch .Version }}-snapshot"

src/internal/cluster/state.go

@@ -183,6 +183,9 @@ func (c *Cluster) sanitizeZarfState(state types.ZarfState) types.ZarfState {
 	sanitizedState.RegistryInfo.PullPassword = "**sanitized**"
 	sanitizedState.RegistryInfo.Secret = "**sanitized**"
 
+	// Overwrite the ArtifactServer secret
+	sanitizedState.ArtifactServer.PushToken = "**sanitized**"
+
 	// Overwrite the Logging secret
 	sanitizedState.LoggingSecret = "**sanitized**"
 

src/test/e2e/20_zarf_init_test.go

@@ -61,6 +61,15 @@ func TestZarfInit(t *testing.T) {
 		require.NoError(t, err)
 	}
 
+	// Check for any old secrets to ensure that they don't get saved in the init log
+	oldState := types.ZarfState{}
+	base64State, _, err := e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}")
+	if err == nil {
+		oldStateJSON, err := base64.StdEncoding.DecodeString(base64State)
+		require.NoError(t, err)
+		err = json.Unmarshal(oldStateJSON, &oldState)
+	}
+
 	// run `zarf init`
 	_, initStdErr, err := e2e.Zarf("init", "--components="+initComponents, "--nodeport", "31337", "-l", "trace", "--confirm")
 	require.NoError(t, err)
@@ -69,23 +78,19 @@ func TestZarfInit(t *testing.T) {
 	logText := e2e.GetLogFileContents(t, initStdErr)
 
 	// Verify that any state secrets were not included in the log
-	base64State, _, err := e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}")
+	state := types.ZarfState{}
+	base64State, _, err = e2e.Kubectl("get", "secret", "zarf-state", "-n", "zarf", "-o", "jsonpath={.data.state}")
 	require.NoError(t, err)
 	stateJSON, err := base64.StdEncoding.DecodeString(base64State)
 	require.NoError(t, err)
-	state := types.ZarfState{}
 	err = json.Unmarshal(stateJSON, &state)
 	require.NoError(t, err)
-	require.NotContains(t, logText, state.AgentTLS.CA)
-	require.NotContains(t, logText, state.AgentTLS.Cert)
-	require.NotContains(t, logText, state.AgentTLS.Key)
-	require.NotContains(t, logText, state.ArtifactServer.PushToken)
-	require.NotContains(t, logText, state.GitServer.PullPassword)
-	require.NotContains(t, logText, state.GitServer.PushPassword)
-	require.NotContains(t, logText, state.RegistryInfo.PullPassword)
-	require.NotContains(t, logText, state.RegistryInfo.PushPassword)
-	require.NotContains(t, logText, state.RegistryInfo.Secret)
-	require.NotContains(t, logText, state.LoggingSecret)
+	checkLogForSensitiveState(t, logText, state)
+
+	// Check the old state values as well (if they exist) to ensure they weren't printed and then updated during init
+	if oldState.LoggingSecret != "" {
+		checkLogForSensitiveState(t, logText, oldState)
+	}
 
 	if e2e.ApplianceMode {
 		// make sure that we upgraded `k3s` correctly and are running the correct version - this should match that found in `packages/distros/k3s`
@@ -108,3 +113,19 @@ func TestZarfInit(t *testing.T) {
 	_, _, _ = e2e.Kubectl("scale", "deploy", "-n", "kube-system", "coredns", "--replicas=1")
 	_, _, _ = e2e.Kubectl("scale", "deploy", "-n", "zarf", "agent-hook", "--replicas=1")
 }
+
+func checkLogForSensitiveState(t *testing.T, logText string, zarfState types.ZarfState) {
+	require.NotContains(t, logText, zarfState.AgentTLS.CA)
+	require.NotContains(t, logText, string(zarfState.AgentTLS.CA))
+	require.NotContains(t, logText, zarfState.AgentTLS.Cert)
+	require.NotContains(t, logText, string(zarfState.AgentTLS.Cert))
+	require.NotContains(t, logText, zarfState.AgentTLS.Key)
+	require.NotContains(t, logText, string(zarfState.AgentTLS.Key))
+	require.NotContains(t, logText, zarfState.ArtifactServer.PushToken)
+	require.NotContains(t, logText, zarfState.GitServer.PullPassword)
+	require.NotContains(t, logText, zarfState.GitServer.PushPassword)
+	require.NotContains(t, logText, zarfState.RegistryInfo.PullPassword)
+	require.NotContains(t, logText, zarfState.RegistryInfo.PushPassword)
+	require.NotContains(t, logText, zarfState.RegistryInfo.Secret)
+	require.NotContains(t, logText, zarfState.LoggingSecret)
+}

src/test/packages/20-mismatched-arch-init/zarf.yaml

@@ -8,26 +8,5 @@ components:
     import:
       path: ../../../../packages/distros/k3s
 
-  # This package moves the injector & registries binaries
-  - name: zarf-injector
-    required: true
-    import:
-      path: ../../../../packages/zarf-registry
-
-  # Creates the temporary seed-registry
-  - name: zarf-seed-registry
-    required: true
-    import:
-      path: ../../../../packages/zarf-registry
-
-  # Creates the permanent registry
-  - name: zarf-registry
-    required: true
-    import:
-      path: ../../../../packages/zarf-registry
-
-  # Creates the pod+git mutating webhook
-  - name: zarf-agent
-    required: true
-    import:
-      path: ../../../../packages/zarf-agent
+  # This test package skips everything else since we explicitly expect 'expectedErrorMessage'
+  # to come from the `k3s` package or from Zarf checking an already existing cluster