Run Next.js repo tests #649
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run Next.js repo tests | |
| on: | |
| workflow_call: | |
| pull_request: | |
| branches: [main] | |
| schedule: | |
| - cron: '0 3 * * *' # Run every day at 3am UTC | |
| workflow_dispatch: | |
| inputs: | |
| versions: | |
| description: 'The versions of Next.js to test against (escape-quoted and comma separated)' | |
| required: false | |
| # TODO(serhalp) Ideally this would simply accept bare quotes but we're having trouble | |
| # parsing that so this will do for now. | |
| default: "\\\"latest\\\"" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| NODE_VERSION: 18.18.0 | |
| PNPM_VERSION: 8.9.0 | |
| NEXT_REPO: vercel/next.js | |
| NEXT_TEST_MODE: deploy | |
| NEXT_JUNIT_TEST_REPORT: true | |
| # In older versions of Next.js both of these need to be set to enable junit reporting | |
| DATADOG_TRACE_NEXTJS_TEST: true | |
| DATADOG_API_KEY: foo | |
| TEST_CONCURRENCY: 2 | |
| NEXT_E2E_TEST_TIMEOUT: 300000 | |
| NEXT_TELEMETRY_DISABLED: 1 | |
| NEXT_SKIP_NATIVE_POSTINSTALL: 1 | |
| TURBO_API: ${{ secrets.TURBO_API }} | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: next-runtime-minimal | |
| TURBO_TEAMID: team_netlify | |
| NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }} | |
| NETLIFY_SITE_ID: 1d5a5c76-d445-4ae5-b694-b0d3f2e2c395 | |
| NEXT_TEST_CONTINUE_ON_ERROR: 1 | |
| next-path: next.js | |
| runtime-path: next-runtime | |
| GH_TOKEN: ${{ github.token }} | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-e2e-tests')) }} | |
| outputs: | |
| version_spec: ${{ steps.set-matrix.outputs.version_spec }} | |
| group: ${{ steps.set-matrix.outputs.group }} | |
| total: ${{ steps.set-matrix.outputs.total }} | |
| steps: | |
| - name: Set Next.js versions to test | |
| id: set-matrix | |
| run: | | |
| if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then | |
| VERSION_SELECTORS=[${{ github.event.inputs.versions }}] | |
| echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT | |
| echo "total=12" >> $GITHUB_OUTPUT | |
| elif [ "${{ github.event_name }}" == "pull_request" ]; then | |
| VERSION_SELECTORS=[\"latest\"] | |
| echo "group=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]" >> $GITHUB_OUTPUT | |
| echo "total=12" >> $GITHUB_OUTPUT | |
| else | |
| VERSION_SELECTORS=[\"latest\",\"canary\",\"13.5.1\"] | |
| echo "group=[1, 2, 3, 4]" >> $GITHUB_OUTPUT | |
| echo "total=4" >> $GITHUB_OUTPUT | |
| fi | |
| VERSION_SPEC="[" | |
| for QUOTED_SELECTOR in $(echo $VERSION_SELECTORS | jq -c '.[]'); do | |
| SELECTOR=$(echo $QUOTED_SELECTOR | xargs) | |
| if [ "${VERSION_SPEC}" != "[" ]; then | |
| VERSION_SPEC+="," | |
| fi | |
| VERSION_SPEC+="{\"selector\":\"$SELECTOR\"" | |
| if [ "$SELECTOR" == "latest" ]; then | |
| VERSION_SPEC+=",\"tag\":\"$(curl -s https://api.github.com/repos/vercel/next.js/releases/latest | jq -r .tag_name)\"" | |
| elif [ "$SELECTOR" == "canary" ]; then | |
| VERSION_SPEC+=",\"tag\":\"$(curl -s https://api.github.com/repos/vercel/next.js/releases | jq -r '.[] | select(.prerelease == true) | .tag_name' | head -n 1)\"" | |
| else | |
| VERSION_SPEC+=",\"tag\":\"v$SELECTOR\"" | |
| fi | |
| VERSION_SPEC+=",\"version\":\"$(npm view next@$SELECTOR version)\"" | |
| VERSION_SPEC+="}" | |
| done | |
| VERSION_SPEC+="]" | |
| echo "version_spec=$VERSION_SPEC" >> $GITHUB_OUTPUT | |
| e2e: | |
| needs: setup | |
| runs-on: ubuntu-latest | |
| name: Test next@${{ matrix.version_spec.selector }} group ${{ matrix.group }}/${{ needs.setup.outputs.total }} | |
| timeout-minutes: 120 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version_spec: ${{ fromJson(needs.setup.outputs.version_spec) }} | |
| group: ${{ fromJson(needs.setup.outputs.group) }} | |
| steps: | |
| - name: checkout Next.js repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.NEXT_REPO }} | |
| path: ${{ env.next-path }} | |
| ref: ${{ matrix.version_spec.tag }} | |
| - name: checkout runtime repo | |
| uses: actions/checkout@v4 | |
| with: | |
| path: ${{ env.runtime-path }} | |
| - name: setup node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: setup pnpm/yarn | |
| run: corepack enable | |
| - name: get pnpm store | |
| id: pnpm-store | |
| run: echo "PATH=$(pnpm store path --silent)" >> ${GITHUB_OUTPUT} | |
| - name: cache pnpm deps | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.pnpm-store.outputs.PATH }} | |
| key: pnpm-store-${{ hashFiles('next.js/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| pnpm-store- | |
| - name: get npm cache | |
| id: npm-cache | |
| run: echo "PATH=$(npm config get cache)" >> ${GITHUB_OUTPUT} | |
| - name: cache npm deps | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ steps.npm-cache.outputs.PATH }} | |
| key: node-cache-${{ hashFiles('next-runtime/package-lock.json') }} | |
| restore-keys: | | |
| node-cache- | |
| - name: patch Next.js | |
| run: | | |
| cp ../${{ env.runtime-path }}/tests/netlify-deploy.ts test/lib/next-modes/ | |
| git apply ../${{ env.runtime-path }}/tests/e2e-utils.patch || git apply ../${{ env.runtime-path }}/tests/e2e-utils-v2.patch | |
| working-directory: ${{ env.next-path }} | |
| - name: install Next.js | |
| run: pnpm install | |
| working-directory: ${{ env.next-path }} | |
| - name: build Next.js | |
| run: pnpm build | |
| working-directory: ${{ env.next-path }} | |
| - name: install swc | |
| run: pnpm add --workspace-root @next/swc-linux-x64-gnu@${{ matrix.version_spec.version }} | |
| working-directory: ${{ env.next-path }} | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v1 | |
| with: | |
| # Should match the `DENO_VERSION_RANGE` from https://github.com/netlify/edge-bundler/blob/main/node/bridge.ts#L17 | |
| deno-version: v1.37.0 | |
| - name: install runtime | |
| run: npm install --ignore-scripts | |
| working-directory: ${{ env.runtime-path }} | |
| - name: build runtime | |
| run: npm run build | |
| working-directory: ${{ env.runtime-path }} | |
| - name: add netlify cli | |
| run: npm install -g netlify-cli | |
| - name: Install Playwright Browsers | |
| run: npx playwright install --with-deps | |
| working-directory: ${{ env.next-path }} | |
| - name: Get test filters | |
| id: test-filters | |
| run: | | |
| # This is when the manifest version was changed | |
| if [ `npx semver -p -r ">=14.0.4-canary.26" ${{ matrix.version_spec.version }}` ]; then | |
| echo "filters=../next-runtime/tests/netlify-e2e.cjs" >> $GITHUB_OUTPUT | |
| echo "skip-retry=../next-runtime/tests/e2e-skip-retry.json" >> $GITHUB_OUTPUT | |
| else | |
| echo "filters=../next-runtime/tests/netlify-e2e-legacy.json" >> $GITHUB_OUTPUT | |
| echo "skip-retry=../next-runtime/tests/e2e-skip-retry-legacy.json" >> $GITHUB_OUTPUT | |
| fi | |
| - name: run tests | |
| env: | |
| NODE_ENV: production | |
| NEXT_EXTERNAL_TESTS_FILTERS: ${{ steps.test-filters.outputs.filters }} | |
| NEXT_TEST_SKIP_RETRY_MANIFEST: ${{ steps.test-filters.outputs.skip-retry }} | |
| run: node run-tests.js -g ${{ matrix.group }}/${{ needs.setup.outputs.total }} -c ${TEST_CONCURRENCY} --type e2e --timings | |
| working-directory: ${{ env.next-path }} | |
| - name: Upload Test Results | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-result-${{matrix.version_spec.selector}}-${{ matrix.group }} | |
| path: ${{ env.next-path }}/test/test-junit-report/*.xml | |
| publish-test-results: | |
| name: 'E2E Test Summary (${{matrix.version_spec.selector}})' | |
| needs: | |
| - e2e | |
| - setup | |
| runs-on: ubuntu-latest | |
| permissions: | |
| checks: write | |
| contents: read | |
| issues: read | |
| if: success() || failure() | |
| strategy: | |
| matrix: | |
| version_spec: ${{ fromJson(needs.setup.outputs.version_spec) }} | |
| steps: | |
| - name: checkout runtime repo | |
| uses: actions/checkout@v4 | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v1 | |
| - name: Download Artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test-result-${{matrix.version_spec.selector}}-* | |
| path: artifacts | |
| - name: Get and format Github issues | |
| if: success() || failure() | |
| run: | | |
| gh issue list --label "Next.js e2e test failure" --json url,body > report/issues.json | |
| deno run -A tools/deno/ghIssues2json.ts tests/test-config.json report/issues.json | |
| - name: Publish Test Report | |
| id: publish-test-results | |
| if: success() || failure() | |
| run: | | |
| echo "slackEvent<<NETLIFY_EOF" >> $GITHUB_OUTPUT | |
| deno run -A tools/deno/junit2slack.ts --dir artifacts --version ${{matrix.version_spec.selector}} --runUrl ${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}} >> $GITHUB_OUTPUT | |
| echo "NETLIFY_EOF" >> $GITHUB_OUTPUT | |
| deno run -A tools/deno/junit2json.ts artifacts ${{ matrix.version_spec.tag }} > report/test-results.json | |
| deno run -A tools/deno/generate-md.ts >> $GITHUB_STEP_SUMMARY | |
| - name: Upload Test JSON | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| # TODO(serhalp) Consider renaming this. It's misleading, since it's just an identifier, | |
| # but it's formatted like a filename, and happens to be almost - but not quite - the | |
| # actual filename. | |
| name: ${{matrix.version_spec.selector}}-test-results.json | |
| path: report/test-results.json | |
| # FIXME(serhalp) Once we recover access to the SquidlifyBot Slack App, reenable this. | |
| # - name: Notify Slack | |
| # if: ${{ (success() || failure()) && github.event_name == 'schedule' }} | |
| # uses: slackapi/slack-github-action@v1.26.0 | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| # SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK | |
| # with: | |
| # payload: ${{ steps.publish-test-results.outputs.slackEvent }} |