diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml new file mode 100644 index 0000000000000..c8657cebadca7 --- /dev/null +++ b/.github/workflows/test-integrations.yml @@ -0,0 +1,379 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +name: test-integrations + +on: + pull_request: + branches-ignore: + - stable-website + - 'docs/**' + - 'ui/**' + - 'mktg-**' # Digital Team Terraform-generated branch prefix + - 'backport/docs/**' + - 'backport/ui/**' + - 'backport/mktg-**' + +env: + TEST_RESULTS_DIR: /tmp/test-results + TEST_RESULTS_ARTIFACT_NAME: test-results + CONSUL_LICENSE: ${{ secrets.CONSUL_LICENSE }} + GOTAGS: ${{ endsWith(github.repository, '-enterprise') && 'consulent' || '' }} + +jobs: + setup: + runs-on: ubuntu-latest + name: Setup + outputs: + compute-small: ${{ steps.runners.outputs.compute-small }} + compute-medium: ${{ steps.runners.outputs.compute-medium }} + compute-large: ${{ steps.runners.outputs.compute-large }} + compute-xl: ${{ steps.runners.outputs.compute-xl }} + enterprise: ${{ steps.runners.outputs.enterprise }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - id: runners + run: .github/scripts/get_runner_classes.sh + + generate-envoy-job-matrices: + needs: [setup] + runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }} + name: Generate Envoy Job Matrices + outputs: + envoy-matrix: ${{ steps.set-matrix.outputs.envoy-matrix }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - name: Generate Envoy Job Matrix + id: set-matrix + env: + # this is further going to multiplied in envoy-integration tests by the + # other dimensions in the matrix. Currently TOTAL_RUNNERS would be + # multiplied by 8 based on these values: + # envoy-version: ["1.22.7", "1.23.4", "1.24.2", "1.25.1"] + # xds-target: ["server", "client"] + TOTAL_RUNNERS: 3 + JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]' + run: | + { + echo -n "envoy-matrix=" + find ./test/integration/connect/envoy -maxdepth 1 -type d -print0 \ + | xargs -0 -n 1 basename \ + | jq --raw-input --argjson runnercount "$TOTAL_RUNNERS" "$JQ_SLICER" \ + | jq --compact-output 'map(join("|"))' + } >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + dev-build: + needs: [setup] + uses: ./.github/workflows/reusable-dev-build.yml + with: + runs-on: ${{ needs.setup.outputs.compute-xl }} + repository-name: ${{ github.repository }} + uploaded-binary-name: 'consul-bin' + secrets: + elevated-github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + + nomad-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + strategy: + matrix: + nomad-version: ['v1.3.3', 'v1.2.10', 'v1.1.16'] + steps: + - name: Setup gotestsum + uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0 + with: + gotestsum_version: 1.9.0 + - name: Checkout Nomad + uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + with: + repository: hashicorp/nomad + ref: ${{ matrix.nomad-version }} + + - name: Install Go + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - name: Fetch Consul binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: 'consul-bin' + path: ./bin + - name: Restore Consul permissions + run: | + chmod +x ./bin/consul + echo "$(pwd)/bin" >> $GITHUB_PATH + + - name: Make Nomad dev build + run: make pkg/linux_amd64/nomad + + - name: Run integration tests + run: | + gotestsum \ + --format=short-verbose \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --packages="./command/agent/consul" \ + --junitfile $TEST_RESULTS_DIR/results.xml -- \ + -run TestConsul + + vault-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-large) }} + needs: + - setup + - dev-build + strategy: + matrix: + vault-version: ["1.12.2", "1.11.6", "1.10.9", "1.9.10"] + env: + VAULT_BINARY_VERSION: ${{ matrix.vault-version }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + + # NOTE: This step is specifically needed for ENT. It allows us to access the required private HashiCorp repos. + - name: Setup Git + if: ${{ endsWith(github.repository, '-enterprise') }} + run: git config --global url."https://${{ secrets.ELEVATED_GITHUB_TOKEN }}:@github.com".insteadOf "https://github.com" + + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + + - uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0 + with: + gotestsum_version: 1.9.0 + + - name: Install Vault + run: | + wget -q -O /tmp/vault.zip "https://releases.hashicorp.com/vault/${{ env.VAULT_BINARY_VERSION }}/vault_${{ env.VAULT_BINARY_VERSION }}_linux_amd64.zip" + unzip -d /tmp /tmp/vault.zip + echo "/tmp" >> $GITHUB_PATH + + - name: Run Connect CA Provider Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + make test-connect-ca-providers + + envoy-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + permissions: + id-token: write # NOTE: this permission is explicitly required for Vault auth. + contents: read + needs: + - setup + - generate-envoy-job-matrices + - dev-build + strategy: + fail-fast: false + matrix: + envoy-version: ["1.22.7", "1.23.4", "1.24.2", "1.25.1"] + xds-target: ["server", "client"] + test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }} + env: + ENVOY_VERSION: ${{ matrix.envoy-version }} + XDS_TARGET: ${{ matrix.xds-target }} + AWS_LAMBDA_REGION: us-west-2 + steps: + # NOTE: ENT specific step as we store secrets in Vault. + - name: Authenticate to Vault + if: ${{ endsWith(github.repository, '-enterprise') }} + id: vault-auth + run: vault-auth + + # NOTE: ENT specific step as we store secrets in Vault. + - name: Fetch Secrets + if: ${{ endsWith(github.repository, '-enterprise') }} + id: secrets + uses: hashicorp/vault-action@v2.5.0 + with: + url: ${{ steps.vault-auth.outputs.addr }} + caCertificate: ${{ steps.vault-auth.outputs.ca_certificate }} + token: ${{ steps.vault-auth.outputs.token }} + secrets: | + kv/data/github/${{ github.repository }}/aws arn | AWS_ROLE_ARN ; + + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - name: Setup gotestsum + uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0 + with: + gotestsum_version: 1.9.0 + + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: 'consul-bin' + path: ./bin + - name: restore mode+x + run: chmod +x ./bin/consul + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v2.4.1 + + - name: Docker build + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile ./bin + + - name: Envoy Integration Tests + env: + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + LAMBDA_TESTS_ENABLED: "true" + # tput complains if this isn't set to something. + TERM: ansi + run: | + # shellcheck disable=SC2001 + echo "Running $(sed 's,|, ,g' <<< "${{ matrix.test-cases }}" |wc -w) subtests" + # shellcheck disable=SC2001 + sed 's,|,\n,g' <<< "${{ matrix.test-cases }}" + gotestsum \ + --debug \ + --rerun-fails \ + --rerun-fails-report=/tmp/gotestsum-rerun-fails \ + --jsonfile /tmp/jsonfile/go-test.log \ + --packages=./test/integration/connect/envoy \ + -- -timeout=30m -tags integration -run="TestEnvoy/(${{ matrix.test-cases }})" + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: container-logs + path: ./test/integration/connect/envoy/workdir/logs + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + compatibility-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + needs: + - setup + - dev-build + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - name: Setup gotestsum + uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0 + with: + gotestsum_version: 1.9.0 + + # Build the consul:local image from the already built binary + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: 'consul-bin' + path: . + - name: restore mode+x + run: chmod +x consul + + - name: Build consul:local image + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Compatibility Integration Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + cd ./test/integration/consul-container + docker run --rm consul:local consul version + # shellcheck disable=SC2046 + gotestsum \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=3 \ + --packages="./..." \ + -- \ + go test \ + -p=4 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json \ + $(go list ./... | grep -v upgrade) \ + --target-image consul \ + --target-version local \ + --latest-image consul \ + --latest-version latest + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and circleci + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }} + + upgrade-integration-test: + runs-on: ${{ fromJSON(needs.setup.outputs.compute-xl) }} + needs: + - setup + - dev-build + strategy: + matrix: + consul-version: [ "1.14", "1.15"] + env: + CONSUL_VERSION: ${{ matrix.consul-version }} + steps: + - uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0 + - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0 + with: + go-version-file: 'go.mod' + - name: Setup gotestsum + uses: autero1/action-gotestsum@2e48af62f5248bd3b014f598cd1aa69a01dd36e3 # v1.0.0 + with: + gotestsum_version: 1.9.0 + + # Get go binary from workspace + - name: fetch binary + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: 'consul-bin' + path: . + - name: restore mode+x + run: chmod +x consul + - name: Build consul:local image + run: docker build -t consul:local -f ./build-support/docker/Consul-Dev.dockerfile . + - name: Upgrade Integration Tests + run: | + mkdir -p "${{ env.TEST_RESULTS_DIR }}" + cd ./test/integration/consul-container/test/upgrade + docker run --rm consul:local consul version + PACKAGE_NAMES=$(go list -tags "${{ env.GOTAGS }}" ./...) + gotestsum \ + --raw-command \ + --format=short-verbose \ + --debug \ + --rerun-fails=3 \ + --packages="${PACKAGE_NAMES}" \ + -- \ + go test \ + -p=8 \ + -tags "${{ env.GOTAGS }}" \ + -timeout=30m \ + -json \ + ./... \ + --target-image consul \ + --target-version local \ + --latest-image consul \ + --latest-version "$CONSUL_VERSION" + ls -lrt + env: + # this is needed because of incompatibility between RYUK container and circleci + GOTESTSUM_JUNITFILE: ${{ env.TEST_RESULTS_DIR }}/results.xml + GOTESTSUM_FORMAT: standard-verbose + COMPOSE_INTERACTIVE_NO_CLI: 1 + # tput complains if this isn't set to something. + TERM: ansi + - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ env.TEST_RESULTS_ARTIFACT_NAME }} + path: ${{ env.TEST_RESULTS_DIR }}