diff --git a/.github/actions/save-logs/action.yaml b/.github/actions/save-logs/action.yaml index fd5a9aa7..f6bfbc14 100644 --- a/.github/actions/save-logs/action.yaml +++ b/.github/actions/save-logs/action.yaml @@ -1,15 +1,20 @@ name: save-logs description: "Save debug logs" +inputs: + name: + description: "unique name to put in log file" + required: true + runs: using: composite steps: - name: Fix log permissions run: | - sudo chown $USER /tmp/zarf-*.log || echo "" + sudo chown $USER /tmp/uds-${{ inputs.name }}.log || echo "" shell: bash - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: debug-log - path: /tmp/zarf-*.log + path: /tmp/uds-${{ inputs.name }}.log diff --git a/.github/actions/setup-from-previous/action.yaml b/.github/actions/setup-from-previous/action.yaml new file mode 100644 index 00000000..b030e2de --- /dev/null +++ b/.github/actions/setup-from-previous/action.yaml @@ -0,0 +1,28 @@ +name: setup-from-previous +description: grabs artifact from a previous job and sets up the env for tests + +runs: + using: composite + steps: + # Checkout the repo and setup the tooling for this job + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + + - name: Download build artifacts + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + name: build-artifacts + path: build/ + + - name: Setup golang + uses: ./.github/actions/golang + + - name: Make UDS-CLI executable + shell: bash + run: | + chmod +x build/uds + + - name: Setup K3d + uses: ./.github/actions/k3d diff --git a/.github/workflows/test-e2e-pr.yaml b/.github/workflows/test-e2e-pr.yaml index c5c90b3d..c3312893 100644 --- a/.github/workflows/test-e2e-pr.yaml +++ b/.github/workflows/test-e2e-pr.yaml @@ -21,11 +21,14 @@ concurrency: cancel-in-progress: true jobs: - test: + build: runs-on: ubuntu-latest steps: + # Checkout the repo and setup the tooling for this job - name: Checkout uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 - name: Setup golang uses: ./.github/actions/golang @@ -33,16 +36,123 @@ jobs: - name: Install UDS CLI uses: ./.github/actions/install-uds-cli - - name: Build UDS-CLI binary - run: uds run build-cli-linux-amd + - name: Build CLI + run: | + uds run build-cli-linux-amd + + # Upload the contents of the build directory for later stages to use + - name: Upload build artifacts + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: build-artifacts + path: build/ + retention-days: 1 + test-bundle: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + + - name: setup-using-previous-job + uses: ./.github/actions/setup-from-previous + + - name: Run e2e bundle tests + run: | + build/uds run test:run-bundle-test --no-progress + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs + with: + name: build + + test-dev: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + + - name: setup-using-previous-job + uses: ./.github/actions/setup-from-previous + + - name: Run e2e dev tests + run: | + build/uds run test:run-dev-test --no-progress + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs + with: + name: test-dev + + test-variables: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + + - name: setup-using-previous-job + uses: ./.github/actions/setup-from-previous + + - name: Run e2e variable tests + run: | + build/uds run test:run-variable-test --no-progress + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs + with: + name: test-variables + + test-optional-bundle: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 + + - name: setup-using-previous-job + uses: ./.github/actions/setup-from-previous + + - name: Run e2e optional bundle tests + run: | + build/uds run test:run-optional-bundle-tests --no-progress + + - name: Save logs + if: always() + uses: ./.github/actions/save-logs + with: + name: test-optional-bundle + + test-vendor: + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout + uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6 + with: + fetch-depth: 0 - - name: Setup K3d - uses: ./.github/actions/k3d + - name: setup-using-previous-job + uses: ./.github/actions/setup-from-previous - - name: Run e2e tests + - name: Run e2e vendor tests run: | - build/uds run test:e2e-no-ghcr-write --no-progress + build/uds run test:run-vendor-tests --no-progress - name: Save logs if: always() uses: ./.github/actions/save-logs + with: + name: test-vendor diff --git a/README.md b/README.md index 85050f1b..f1679521 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,6 @@ [![Build Status](https://img.shields.io/github/actions/workflow/status/defenseunicorns/uds-cli/release.yaml)](https://github.com/defenseunicorns/uds-cli/actions/workflows/release.yaml) [![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/defenseunicorns/uds-cli/badge)](https://api.securityscorecards.dev/projects/github.com/defenseunicorns/uds-cli) -**:warning: Warning**: UDS-CLI is in early alpha, expect changes to the schema and workflow - ## Table of Contents 1. [Install](#install) diff --git a/src/test/e2e/bundle_test.go b/src/test/e2e/bundle_test.go index 6520c150..ae225d25 100644 --- a/src/test/e2e/bundle_test.go +++ b/src/test/e2e/bundle_test.go @@ -20,12 +20,6 @@ import ( "github.com/defenseunicorns/uds-cli/src/config" ) -func zarfPublish(t *testing.T, path string, reg string) { - args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10 -l debug", path, reg), " ") - _, _, err := e2e.UDS(args...) - require.NoError(t, err) -} - func TestUDSCmd(t *testing.T) { _, _, err := e2e.UDS() require.NoError(t, err) diff --git a/src/test/e2e/commands_test.go b/src/test/e2e/commands_test.go index 5a6aa280..86e6a72c 100644 --- a/src/test/e2e/commands_test.go +++ b/src/test/e2e/commands_test.go @@ -25,6 +25,12 @@ import ( // This file contains helpers for running UDS CLI commands (ie. uds create/deploy/etc with various flags and options) +func zarfPublish(t *testing.T, path string, reg string) { + args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10 -l debug --no-progress", path, reg), " ") + _, _, err := e2e.UDS(args...) + require.NoError(t, err) +} + func createLocal(t *testing.T, bundlePath string, arch string) { cmd := strings.Split(fmt.Sprintf("create %s --insecure --confirm -a %s", bundlePath, arch), " ") _, _, err := e2e.UDS(cmd...) diff --git a/src/test/e2e/runner_inputs_test.go b/src/test/e2e/runner_inputs_test.go deleted file mode 100644 index e0a456a1..00000000 --- a/src/test/e2e/runner_inputs_test.go +++ /dev/null @@ -1,119 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2023-Present The UDS Authors - -// Package test provides e2e tests for UDS. -package test - -import ( - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestRunnerInputs(t *testing.T) { - t.Run("test that default values for inputs work when not required", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "has-default-empty", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "default") - require.NotContains(t, stdErr, "{{") - }) - - t.Run("test that default values for inputs work when required", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "has-default-and-required-empty", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "default") - require.NotContains(t, stdErr, "{{") - - }) - - t.Run("test that default values for inputs work when required and have values supplied", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "has-default-and-required-supplied", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "supplied-value") - require.NotContains(t, stdErr, "{{") - }) - - t.Run("test that inputs that aren't required with no default don't error", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "no-default-empty", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.NotContains(t, stdErr, "has-no-default") - require.NotContains(t, stdErr, "{{") - }) - - t.Run("test that inputs with no defaults that aren't required don't error when supplied with a value", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "no-default-supplied", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "success + supplied-value") - require.NotContains(t, stdErr, "{{") - }) - - t.Run("test that tasks that require inputs with no defaults error when called without values", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "no-default-and-required-empty", "--file", "src/test/tasks/inputs/tasks.yaml") - require.Error(t, err, stdOut, stdErr) - require.NotContains(t, stdErr, "{{") - }) - - t.Run("test that tasks that require inputs with no defaults run when supplied with a value", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "no-default-and-required-supplied", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "supplied-value") - require.NotContains(t, stdErr, "{{") - }) - - t.Run("test that when a task is called with extra inputs it warns", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "no-default-and-required-supplied-extra", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "supplied-value") - require.Contains(t, stdErr, "WARNING") - require.Contains(t, stdErr, "does not have an input named extra") - require.NotContains(t, stdErr, "{{") - }) - - t.Run("test that displays a deprecated message", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "deprecated-task", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "WARNING") - require.Contains(t, stdErr, "This input has been marked deprecated: This is a deprecated message") - }) - - t.Run("test that variables can be used as inputs", func(t *testing.T) { - t.Parallel() - - stdOut, stdErr, err := e2e.UDS("run", "variable-as-input", "--file", "src/test/tasks/inputs/tasks.yaml", "--set", "foo=im a variable") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "im a variable") - }) - - t.Run("test that env vars can be used as inputs and take precedence over default vals", func(t *testing.T) { - os.Setenv("UDS_FOO", "im an env var") - stdOut, stdErr, err := e2e.UDS("run", "variable-as-input", "--file", "src/test/tasks/inputs/tasks.yaml") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "im an env var") - }) - - t.Run("test that a --set var has the greatest precedence for inputs", func(t *testing.T) { - os.Setenv("UDS_FOO", "im an env var") - stdOut, stdErr, err := e2e.UDS("run", "variable-as-input", "--file", "src/test/tasks/inputs/tasks.yaml", "--set", "foo=most specific") - require.NoError(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "most specific") - }) -} diff --git a/tasks/tests.yaml b/tasks/tests.yaml index 647c7411..95916b54 100644 --- a/tasks/tests.yaml +++ b/tasks/tests.yaml @@ -15,17 +15,41 @@ tasks: - name: e2e-no-ghcr-write description: run e2e tests without writing to GHCR actions: - - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m -skip ".*GHCR.*" + - task: run-bundle-test + - task: run-dev-test + - task: run-variable-test + - task: run-optional-bundle-tests + - task: run-vendor-tests - name: e2e-ghcr description: run e2e tests that write to GHCR actions: - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m -run ".*GHCR.*" - - name: e2e-runner - description: run e2e tests for task runner only + - name: run-bundle-test + description: only run tests in bundle_test.go actions: - - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m -run TestTaskRunner + - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m bundle_test.go commands_test.go main_test.go + + - name: run-dev-test + description: only run tests in dev.go + actions: + - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m dev_test.go commands_test.go main_test.go + + - name: run-variable-test + description: only run tests in variable_test.go + actions: + - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m variable_test.go commands_test.go main_test.go + + - name: run-optional-bundle-tests + description: only run tests in optional_bundle_test.go + actions: + - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m optional_bundle_test.go commands_test.go main_test.go + + - name: run-vendor-tests + description: only run tests in zarf_test.go and runner_test.go + actions: + - cmd: cd src/test/e2e && go test -failfast -v -timeout 30m zarf_test.go runner_test.go commands_test.go main_test.go - name: push-test-artifacts description: push artifacts that UDS CLI tests rely on to GHCR