run-tests.yml

name: 'Run tests'
on:
  pull_request:
    branches: [main]
  schedule:
    - cron: '0 6 * * *' # Run every day at 6am UTC
  workflow_dispatch:
    inputs:
      versions:
        description: 'The versions of Next.js to test against (quoted and comma separated)'
        required: false
        default: 'latest'

jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Check PR labels
        if: github.event_name == 'pull_request'
        id: check-labels
        uses: actions/github-script@v7
        with:
          script: |
            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.payload.pull_request.number,
            });
            return labels.some(label => label.name === 'autorelease: pending' || label.name === 'test all versions');
      - name: Set Next.js versions to test
        id: set-matrix
        # If this is the nightly build or a release PR then run the full matrix of versions
        run: |
          if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo "matrix=${{ github.event.inputs.versions }}" >> $GITHUB_OUTPUT
          elif [ "${{ github.event_name }}" = "schedule" ] || [ "${{ steps.check-labels.outputs.result }}" = "true" ]; then
            echo "matrix=[\"latest\", \"canary\", \"13.5.1\"]" >> $GITHUB_OUTPUT
          else
            echo "matrix=[\"latest\"]" >> $GITHUB_OUTPUT
          fi
  e2e:
    needs: setup
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        version: ${{ fromJson(needs.setup.outputs.matrix) }}
        shard: [1, 2, 3, 4, 5]

    steps:
      - uses: actions/checkout@v4
      - name: 'Install Node'
        uses: actions/setup-node@v4
        with:
          node-version: '18.x'
          cache: 'npm'
          cache-dependency-path: '**/package-lock.json'
      - uses: oven-sh/setup-bun@v1
      - name: setup pnpm/yarn
        run: corepack enable
        shell: bash
      - name: Install Deno
        uses: denoland/setup-deno@v1
        with:
          # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/main/node/bridge.ts#L17
          deno-version: v1.37.0
      - name: 'Install dependencies'
        run: npm ci
      - name: 'Netlify Login'
        env:
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
        run: |
          # TODO(serhalp) Unpin version once
          # https://netlify.slack.com/archives/C07686YAY13/p1718139257978049 is resolved.
          npm i -g netlify-cli@17.26.0
          netlify login
      - name: Get installed Playwright version
        id: playwright-version
        run: echo "version=$(npm view @playwright/test version)" >> $GITHUB_OUTPUT
      - uses: actions/cache@v4
        id: playwright-cache
        with:
          path: '~/.cache/ms-playwright'
          key: '${{ runner.os }}-playwright-${{ steps.playwright-version.outputs.version }}'
          restore-keys: |
            ${{ runner.os }}-playwright-
      - name: Install Playwright Browsers
        if: steps.playwright-cache.outputs.cache-hit != 'true'
        run: npx playwright install --with-deps
      - name: Resolve Next.js version
        id: resolve-next-version
        shell: bash
        run: |
          RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version)
          echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT
          echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'"
      - name: Run Playwright tests
        run: npm run test:ci:e2e -- --shard=${{ matrix.shard }}/5
        env:
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
          NEXT_VERSION: ${{ matrix.version }}
          NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
      - name: Upload blob report to GitHub Actions Artifacts
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: blob-report-${{matrix.version}}-${{ matrix.shard }}
          path: blob-report
          retention-days: 1

  test:
    needs: setup
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4, 5, 6, 7, 8]
        os: [ubuntu-latest, windows-latest]
        version: ${{ fromJson(needs.setup.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4
      - name: 'Install Node'
        uses: actions/setup-node@v4
        with:
          node-version: '18.x'
          cache: 'npm'
          cache-dependency-path: '**/package-lock.json'
      - name: setup pnpm/yarn
        run: corepack enable
        shell: bash
      - name: Install Deno
        uses: denoland/setup-deno@v1
        with:
          # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/e55f825bd985d3c92e21d1b765d71e70d5628fba/node/bridge.ts#L17
          deno-version: v1.37.0
      - name: 'Install dependencies'
        run: npm ci
      - name: 'Build'
        run: npm run build
      - name: Resolve Next.js version
        id: resolve-next-version
        shell: bash
        run: |
          RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version)
          echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT
          echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'"
      - name: Compute Fixtures Cache Key
        id: fixture-cache-key
        # Fixtures only need to be rebuilt if either fixture or support files change,
        # so we're using a hash of the fixtures and support files as the cache key.
        run:
          echo "key=$(git ls-files -s tests/fixtures/ tests/utils/ tests/prepare.mjs | git hash-object --stdin)"
          >> "$GITHUB_OUTPUT"
        shell: bash
      - name: Cache Fixtures
        id: cache-fixtures
        uses: actions/cache@v4
        with:
          path: tests/fixtures
          key:
            integration-fixtures-${{ runner.os }}-${{steps.resolve-next-version.outputs.version}}-${{
            steps.fixture-cache-key.outputs.key }}
      - name: 'Prepare Fixtures'
        if: steps.cache-fixtures.outputs.cache-hit != 'true'
        run: npm run pretest
        env:
          NEXT_VERSION: ${{ matrix.version }}
          NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
      - name: 'Unit and integration tests'
        run: npm run test:ci:unit-and-integration -- --shard=${{ matrix.shard }}/8
        env:
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
          NEXT_VERSION: ${{ matrix.version }}
          NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}
          TEMP: ${{ github.workspace }}/..

  smoke:
    if: always()
    needs: setup
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        version: ${{ fromJson(needs.setup.outputs.matrix) }}
    steps:
      - uses: actions/checkout@v4
      - name: 'Install Node'
        uses: actions/setup-node@v4
        with:
          node-version: '18.x'
          cache: 'npm'
          cache-dependency-path: '**/package-lock.json'
      - name: setup pnpm/yarn
        run: corepack enable
        shell: bash
      - name: Install Deno
        uses: denoland/setup-deno@v1
        with:
          # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/main/node/bridge.ts#L17
          deno-version: v1.37.0
      - name: 'Install dependencies'
        run: npm ci
      - name: 'Build'
        run: npm run build
      - name: 'Netlify Login'
        env:
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
        run: |
          # TODO(serhalp) Unpin version once
          # https://netlify.slack.com/archives/C07686YAY13/p1718139257978049 is resolved.
          npm i -g netlify-cli@17.26.0
          netlify login
      - name: Resolve Next.js version
        id: resolve-next-version
        shell: bash
        run: |
          RESOLVED_VERSION=$(npm view next@${{ matrix.version }} version)
          echo "version=$RESOLVED_VERSION" >> $GITHUB_OUTPUT
          echo "Resolved Next.js version for 'next@${{ matrix.version }}' is '$RESOLVED_VERSION'"
      - name: 'Smoke tests'
        run: npm run test:ci:smoke
        env:
          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
          NEXT_VERSION: ${{ matrix.version }}
          NEXT_RESOLVED_VERSION: ${{ steps.resolve-next-version.outputs.version }}

  merge-reports:
    if: always()
    needs: [setup,e2e]
    strategy:
      fail-fast: false
      matrix:
        version: ${{ fromJson(needs.setup.outputs.matrix) }}

    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: actions/setup-node@v4
      with:
        node-version: 18
    - name: Install dependencies
      run: npm ci

    - name: Download blob reports from GitHub Actions Artifacts
      uses: actions/download-artifact@v4
      with:
        path: all-blob-reports
        pattern: blob-report-${{ matrix.version }}-*
        merge-multiple: true

    - name: Merge reports
      run: |
        npx playwright merge-reports --reporter html ./all-blob-reports
        npx playwright merge-reports --reporter json ./all-blob-reports > merged_reports.json

    # FIXME(serhalp) Once we recover access to the SquidlifyBot Slack App, reenable this.
    # - name: Notify Slack
    #   if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.head_ref == 'main'
    #   run: npx playwright-slack-report -j merged_reports.json -c tests/playwright-slack-conf.json
    #   env:
    #     SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }}
    #     RESULTS_VERSION: ${{ matrix.version }}
    #     RESULTS_URL: ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}

    - name: Upload HTML report
      uses: actions/upload-artifact@v4
      with:
        name: html-report-${{ matrix.version }}-attempt-${{ github.run_attempt }}
        path: playwright-report
        retention-days: 14