BATS #168
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: BATS | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| owner: | |
| description: Override owner (e.g. rancher-sandbox) | |
| type: string | |
| repo: | |
| description: Override repository (e.g. rancher-desktop) | |
| type: string | |
| branch: | |
| description: Override branch (e.g. main, or PR#) | |
| type: string | |
| tests: | |
| description: 'Tests (in the tests/ directory, e.g. "containers")' | |
| default: '*' | |
| type: string | |
| platforms: | |
| description: Platforms to run | |
| default: 'linux mac win' | |
| type: string | |
| engines: | |
| description: Container engines to run | |
| default: 'containerd moby' | |
| type: string | |
| kubernetes-version: | |
| description: Primary Kubernetes version to test | |
| default: '1.22.7' # Must also change in calculate step | |
| type: string | |
| kubernetes-alt-version: | |
| description: Secondary Kubernetes version to test (e.g. for upgrades) | |
| default: '1.28.11' # Must also change in calculate step | |
| type: string | |
| package-id: | |
| description: Package run ID override; leave empty to use latest. | |
| default: '' | |
| type: string | |
| experimental: | |
| description: Run with experimental settings (WSL) | |
| default: false | |
| type: boolean | |
| schedule: | |
| - cron: '0 8 * * 1-5' # 8AM UTC weekdays as a baseline | |
| permissions: | |
| contents: read | |
| env: | |
| GH_OWNER: ${{ github.repository_owner }} | |
| GH_REPOSITORY: ${{ github.repository }} | |
| GH_REF_NAME: ${{ github.ref_name }} | |
| jobs: | |
| get-tests: | |
| name: Calculate tests to run | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fetch install script | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout-cone-mode: false | |
| sparse-checkout: | | |
| scripts/install-latest-ci.sh | |
| .github/workflows/bats/get-tests.py | |
| - id: repo | |
| name: Calculate short repository | |
| run: echo "repo=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_OUTPUT" | |
| - name: Fetch tests | |
| run: | | |
| : ${OWNER:=$GH_OWNER} | |
| : ${REPO:=${GH_REPOSITORY#$GH_OWNER/}} | |
| : ${BRANCH:=$GH_REF_NAME} | |
| # If BRANCH is a number, assume it is supposed to be a PR | |
| [[ $BRANCH =~ ^[0-9]+$ ]] && export PR=$BRANCH | |
| "scripts/install-latest-ci.sh" | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ inputs.owner || github.repository_owner }} | |
| REPO: ${{ inputs.repo || steps.repo.outputs.repo }} | |
| BRANCH: ${{ inputs.branch || github.ref_name }} | |
| ID: ${{ inputs.package-id }} | |
| BATS_DIR: ${{ github.workspace }}/bats | |
| INSTALL_MODE: skip | |
| - name: Calculate tests | |
| id: calculate | |
| # This script is not inline to make local testing easier | |
| run: python3 ${{ github.workspace }}/.github/workflows/bats/get-tests.py | |
| env: | |
| TESTS: ${{ inputs.tests }} | |
| PLATFORMS: ${{ inputs.platforms }} | |
| ENGINES: ${{ inputs.engines }} | |
| KUBERNETES_VERSION: ${{ inputs.kubernetes-version || '1.22.7' }} | |
| # rancher/rancher helm chart 2.8.5 supports up to 1.28.* | |
| KUBERNETES_ALT_VERSION: ${{ inputs.kubernetes-alt-version || '1.28.11' }} | |
| working-directory: bats/tests | |
| outputs: | |
| repo: ${{ steps.repo.outputs.repo }} | |
| tests: ${{ steps.calculate.outputs.tests }} | |
| bats: | |
| needs: get-tests | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.get-tests.outputs.tests )}} | |
| runs-on: ${{ matrix.host }} | |
| steps: | |
| - name: Fetch install script | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout-cone-mode: false | |
| sparse-checkout: | | |
| scripts/install-latest-ci.sh | |
| .github/actions/setup-environment/action.yaml | |
| .github/workflows/bats/sanitize-artifact-name.sh | |
| - name: Install latest CI build | |
| run: | | |
| : ${OWNER:=$GH_OWNER} | |
| : ${REPO:=${GH_REPOSITORY#$GH_OWNER/}} | |
| : ${BRANCH:=$GH_REF_NAME} | |
| # If BRANCH is a number, assume it is supposed to be a PR | |
| [[ $BRANCH =~ ^[0-9]+$ ]] && export PR=$BRANCH | |
| scripts/install-latest-ci.sh | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| OWNER: ${{ inputs.owner || github.repository_owner }} | |
| REPO: ${{ inputs.repo || needs.get-tests.outputs.repo }} | |
| BRANCH: ${{ inputs.branch || github.ref_name }} | |
| ID: ${{ inputs.package-id }} | |
| BATS_DIR: ${{ github.workspace }}/bats | |
| INSTALL_MODE: installer | |
| ZIP_NAME: ${{ github.workspace }}/version.txt | |
| RD_LOCATION: system | |
| - name: Set up environment | |
| uses: ./.github/actions/setup-environment | |
| - name: "Linux: Install prerequisites" | |
| if: runner.os == 'Linux' | |
| run: >- | |
| sudo DEBIAN_FRONTEND=noninteractive | |
| apt-get install coreutils | |
| - name: "macOS: Install prerequisites" | |
| if: runner.os == 'macOS' | |
| shell: bash | |
| run: brew install --force bash coreutils | |
| - name: "Windows: Install WSL2 Distribution" | |
| if: runner.os == 'Windows' | |
| shell: powershell # pwsh doesn't have Add-AppxPackage | |
| run: | | |
| # `wsl --install` seems to have issues in CI since 2024-06-14; however, | |
| # manually downloading the Debian installer and running it works. | |
| Invoke-WebRequest https://github.com/microsoft/WSL/raw/master/distributions/DistributionInfo.json | | |
| Select-Object -ExpandProperty Content | | |
| ConvertFrom-JSON | | |
| Select-Object -ExpandProperty Distributions | | |
| Where-Object Name -EQ "Debian" | | |
| Select-Object -ExpandProperty Amd64PackageUrl | | |
| % { Invoke-WebRequest $_ -OutFile Debian.AppxBundle } | |
| Add-AppxPackage Debian.AppxBundle | |
| # Initialize Debian, without going through any first-time setup | |
| debian.exe install --root | |
| - name: "Windows: Install prerequisites in WSL" | |
| if: runner.os == 'Windows' | |
| shell: pwsh | |
| run: >- | |
| wsl.exe -d Debian --exec /usr/bin/env DEBIAN_FRONTEND=noninteractive | |
| sh -c 'apt-get update && apt-get install --yes bsdextrautils coreutils curl' | |
| - name: "Windows: Enable experimental WSL settings" | |
| if: runner.os == 'Windows' && inputs.experimental | |
| shell: pwsh | |
| run: | | |
| Set-Content -Encoding UTF8NoBOM -Path "${HOME}/.wslconfig" -Value @" | |
| ; Note that not all settings here make sense together. | |
| [wsl] | |
| dnsProxy=false | |
| ; networkingMode=mirrored ; https://github.com/rancher-sandbox/rancher-desktop/issues/6665 | |
| firewall=true | |
| dnsTunneling=true | |
| autoProxy=true | |
| [experimental] | |
| autoMemoryReclaim=gradual | |
| sparseVhd=true | |
| useWindowsDnsCache=true | |
| bestEffortDnsParsing=true | |
| hostAddressLoopback=true | |
| "@ | |
| - name: Set log directory | |
| shell: bash | |
| run: | | |
| echo "LOGS_DIR=$(pwd)/logs" >> "$GITHUB_ENV" | |
| mkdir logs | |
| - name: "Windows: Override log directory" | |
| if: runner.os == 'Windows' | |
| shell: powershell | |
| run: >- | |
| wsl.exe -d Debian -- echo 'LOGS_DIR=$(pwd)' | |
| | Out-File -Encoding ASCII -Append "$ENV:GITHUB_ENV" | |
| working-directory: logs | |
| - name: Normalize test name | |
| id: normalize | |
| shell: bash | |
| run: | | |
| t="${{ matrix.name }}" | |
| if [[ ! -r "tests/$t" ]] && [[ -r "tests/${t}.bats" ]]; then | |
| t="${t}.bats" | |
| fi | |
| echo "test=$t" >> "$GITHUB_OUTPUT" | |
| working-directory: bats | |
| - name: "macOS: Set startup command" | |
| if: runner.os == 'macOS' | |
| run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV" | |
| env: | |
| BATS_COMMAND: exec | |
| - name: "Linux: Set startup command" | |
| if: runner.os == 'Linux' | |
| run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV" | |
| env: | |
| BATS_COMMAND: >- | |
| exec xvfb-run --auto-servernum | |
| --server-args='-screen 0 1280x960x24' | |
| - name: "Windows: Set startup command" | |
| if: runner.os == 'Windows' | |
| shell: bash | |
| run: echo "BATS_COMMAND=$BATS_COMMAND" >> "$GITHUB_ENV" | |
| env: | |
| BATS_COMMAND: wsl.exe -d Debian --exec | |
| - name: Run BATS | |
| # We use ${{ env.BATS_COMMAND }} instead of ${BATS_COMMAND} to let the | |
| # shell parse the command, instead of doing it via expansion which is then | |
| # parsed differently (--server-args isn't kept as one word). Also, we | |
| # need to use the env.* form because PowerShell uses ${ENV:VAR} instead. | |
| run: >- | |
| ${{ env.BATS_COMMAND }} | |
| ./bats-core/bin/bats | |
| --gather-test-outputs-in '${{ env.LOGS_DIR }}' | |
| --print-output-on-failure | |
| --filter-tags '!ci-skip' | |
| --formatter cat | |
| --report-formatter tap | |
| 'tests/${{ steps.normalize.outputs.test }}' | |
| env: | |
| BATS_COMMAND: ${{ env.BATS_COMMAND }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| LOGS_DIR: ${{ env.LOGS_DIR }} | |
| RD_CAPTURE_LOGS: "true" | |
| RD_CONTAINER_ENGINE: ${{ matrix.engine }} | |
| RD_KUBERNETES_VERSION: ${{ matrix.k3sVersion }} | |
| RD_KUBERNETES_ALT_VERSION: ${{ matrix.k3sAltVersion }} | |
| RD_TAKE_SCREENSHOTS: "true" | |
| RD_TRACE: "true" | |
| RD_USE_GHCR_IMAGES: "true" | |
| RD_USE_RAMDISK: "true" | |
| RD_USE_WINDOWS_EXE: "${{ runner.os == 'Windows' }}" | |
| WSLENV: "\ | |
| GITHUB_TOKEN:\ | |
| RD_CAPTURE_LOGS:\ | |
| RD_CONTAINER_ENGINE:\ | |
| RD_KUBERNETES_VERSION:\ | |
| RD_KUBERNETES_ALT_VERSION:\ | |
| RD_TAKE_SCREENSHOTS:\ | |
| RD_TRACE:\ | |
| RD_USE_GHCR_IMAGES:\ | |
| RD_USE_RAMDISK:\ | |
| RD_USE_WINDOWS_EXE:\ | |
| " | |
| working-directory: bats | |
| timeout-minutes: 120 | |
| - name: Calculate log name | |
| id: log_name | |
| if: ${{ !cancelled() }} | |
| run: | | |
| name="$(.github/workflows/bats/sanitize-artifact-name.sh <<< "$name")" | |
| # For the artifact name, backslash and forward slash are also invalid. | |
| name=${name//\\/%3C} | |
| name=${name//\//%2F} | |
| echo "name=$name" >>"$GITHUB_OUTPUT" | |
| shell: bash | |
| env: | |
| name: ${{ matrix.host }}-${{ matrix.engine }}-${{ matrix.name }}.logs | |
| - name: Consolidate logs | |
| if: ${{ !cancelled() }} | |
| run: | | |
| # bats/logs may not exist if the workflow is being tested with e.g. tests/helpers/utils.bats | |
| if [ -d "bats/logs" ]; then | |
| cp -R "bats/logs/" logs | |
| fi | |
| cp "bats/report.tap" logs | |
| .github/workflows/bats/sanitize-artifact-name.sh logs | |
| echo "$NAME" > logs/name.txt | |
| echo "$OS" > logs/os.txt | |
| echo "$ENGINE" > logs/engine.txt | |
| echo "$LOG_NAME" > logs/log-name.txt | |
| mv version.txt logs/ | |
| shell: bash | |
| env: | |
| NAME: ${{ matrix.name }} | |
| OS: ${{ matrix.host }} | |
| ENGINE: ${{ matrix.engine }} | |
| LOG_NAME: ${{ steps.log_name.outputs.name }} | |
| - name: Upload logs | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.log_name.outputs.name }} | |
| path: logs/ | |
| if-no-files-found: error | |
| summarize: | |
| name: Summarize output | |
| needs: [ get-tests, bats ] | |
| if: ${{ !cancelled() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Fetch summarizer script | |
| uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| sparse-checkout-cone-mode: false | |
| sparse-checkout: | | |
| package.json | |
| .github/workflows/bats/summarize.mjs | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version-file: package.json | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: "*.logs" | |
| - run: node .github/workflows/bats/summarize.mjs | |
| env: | |
| EXPECTED_TESTS: ${{ needs.get-tests.outputs.tests }} |