add exclusion list

vercel · Jun 11, 2024 · 7dd5585 · 7dd5585
1 parent 64c3279
commit 7dd5585
Show file tree

Hide file tree

Showing 3 changed files with 140 additions and 84 deletions.
diff --git a/.github/workflows/build_reusable.yml b/.github/workflows/build_reusable.yml
@@ -49,6 +49,16 @@ on:
         required: true
         description: 'name of the step, to be used for the upload artifact unique key '
         type: string
+      timeout_minutes:
+        description: 'Timeout in minutes'
+        required: false
+        type: number
+        default: 30
+      runs_on_labels:
+        description: 'List of runner labels'
+        required: false
+        type: string
+        default: '["self-hosted", "linux", "x64", "metal"]'
 
 env:
   NAPI_CLI_VERSION: 2.14.7
@@ -72,12 +82,8 @@ env:
 
 jobs:
   build:
-    timeout-minutes: 30
-    runs-on:
-      - 'self-hosted'
-      - 'linux'
-      - 'x64'
-      - 'metal'
+    timeout-minutes: ${{ inputs.timeout_minutes }}
+    runs-on: ${{ fromJson(inputs.runs_on_labels) }}
 
     outputs:
       input_step_key: ${{ steps.var.outputs.input_step_key }}
@@ -90,7 +96,17 @@ jobs:
           script: |
             core.setOutput('input_step_key', '${{ inputs.stepName }}'.toLowerCase().replaceAll(/[/.]/g, '-').trim('-'));
 
-      - run: fnm use --install-if-missing ${{ inputs.nodeVersion || env.NODE_LTS_VERSION }}
+      - name: Install fnm if needed
+        run: |
+          if ! command -v fnm &> /dev/null
+          then
+            echo "fnm could not be found, installing..."
+            curl -fsSL https://fnm.vercel.app/install | bash
+            echo 'export PATH=$HOME/.fnm:$PATH' >> $HOME/.bashrc
+          else
+            echo "fnm is already installed."
+          fi
+      - run: source $HOME/.bashrc && fnm use --install-if-missing ${{ inputs.nodeVersion || env.NODE_LTS_VERSION }}
       - run: node -v
       - run: corepack enable
       - run: pwd
@@ -175,7 +191,7 @@ jobs:
       - run: turbo run get-test-timings -- --build ${{ github.sha }}
 
       - run: /bin/bash -c "${{ inputs.afterBuild }}"
-        timeout-minutes: 30
+        timeout-minutes: ${{ inputs.timeout_minutes }}
 
       - name: Upload artifact
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/test_e2e_deploy_scheduled.yml b/.github/workflows/test_e2e_deploy_scheduled.yml
@@ -7,84 +7,47 @@ on:
   # allow triggering manually as well
   workflow_dispatch:
 
+env:
+  CARGO_PROFILE_RELEASE_LTO: 'true'
+  DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
+  DD_ENV: 'ci'
+  NAPI_CLI_VERSION: 2.16.2
+  NEXT_JUNIT_TEST_REPORT: 'true'
+  NEXT_TELEMETRY_DISABLED: 1
+  NEXT_TEST_CONTINUE_ON_ERROR: 1
+  NEXT_TEST_JOB: 1
+  NEXT_TEST_MODE: 'deploy'
+  NODE_LTS_VERSION: 20
+  TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
+  TURBO_REMOTE_ONLY: 'true'
+  TURBO_TEAM: 'vercel'
+  TURBO_VERSION: 1.13.3-canary.2
+  VERCEL_TEST_TEAM: vtest314-next-e2e-tests
+  VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
+
 jobs:
-  test:
+  build-next:
     if: github.repository_owner == 'vercel'
-    runs-on: ubuntu-latest
-
-    env:
-      CARGO_PROFILE_RELEASE_LTO: 'true'
-      DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
-      DD_ENV: 'ci'
-      NAPI_CLI_VERSION: 2.16.2
-      NEXT_JUNIT_TEST_REPORT: 'true'
-      NEXT_TELEMETRY_DISABLED: 1
-      NEXT_TEST_CONTINUE_ON_ERROR: 1
-      NEXT_TEST_JOB: 1
-      NEXT_TEST_MODE: 'deploy'
-      NODE_LTS_VERSION: 20
-      TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
-      TURBO_REMOTE_ONLY: 'true'
-      TURBO_TEAM: 'vercel'
-      TURBO_VERSION: 1.13.3-canary.2
-      VERCEL_TEST_TEAM: vtest314-next-e2e-tests
-      VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
-
+    name: build-next
+    uses: ./.github/workflows/build_reusable.yml
+    with:
+      skipNativeBuild: 'yes'
+      stepName: 'build-next'
+      runs_on_labels: '["ubuntu-latest"]'
+      afterBuild: node scripts/run-e2e-test-project-reset.mjs
+    secrets: inherit
+
+  test-deploy:
+    name: test deploy
+    needs: ['build-next']
+    uses: ./.github/workflows/build_reusable.yml
+    secrets: inherit
     strategy:
       fail-fast: false
       matrix:
         group: [1/5, 2/5, 3/5, 4/5, 5/5]
-
-    steps:
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ env.NODE_LTS_VERSION }}
-          check-latest: true
-
-      - name: Setup pnpm
-        run: corepack enable
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 25
-
-      - name: Setup tests
-        run: |
-          pnpm install
-          pnpm run build
-          npm i -g vercel@latest
-          node scripts/run-e2e-test-project-reset.mjs
-
-      - name: Run tests
-        run: |
-          docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.41.2-jammy /bin/bash -c "cd /work && \
-          NODE_VERSION=${NODE_LTS_VERSION} ./scripts/setup-node.sh && \ 
-          corepack enable > /dev/null && \
-          NEXT_JUNIT_TEST_REPORT=${NEXT_JUNIT_TEST_REPORT} \
-          NEXT_TELEMETRY_DISABLED=${NEXT_TELEMETRY_DISABLED} \
-          DATADOG_API_KEY=${DATADOG_API_KEY} \
-          DD_ENV=${DD_ENV} \ 
-          VERCEL_TEST_TOKEN=${VERCEL_TEST_TOKEN} \
-          VERCEL_TEST_TEAM=${VERCEL_TEST_TEAM} \
-          NEXT_TEST_JOB=${NEXT_TEST_JOB} \
-          NEXT_TEST_MODE=${NEXT_TEST_MODE} \
-          TEST_TIMINGS_TOKEN=${TEST_TIMINGS_TOKEN} \
-          NEXT_TEST_CONTINUE_ON_ERROR=${NEXT_TEST_CONTINUE_ON_ERROR} \
-          NEXT_EXTERNAL_TESTS_FILTERS=./test/deploy-tests-manifest.json \
-          xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }} -c 1 >> /proc/1/fd/1"
-
-      - name: Save test report as artifacts
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: test-reports
-          if-no-files-found: ignore
-          retention-days: 2
-          path: test/test-junit-report
-
-      - name: Upload test report to Datadog
-        continue-on-error: true
-        run: |
-          pnpx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:nextjs_deploy_e2e --service nextjs ./test/test-junit-report
+    with:
+      afterBuild: NEXT_TEST_MODE=deploy node run-tests.js --timings -g ${{ matrix.group }} -c 1 --type e2e
+      stepName: 'test-prod-${{ matrix.group }}'
+      timeout_minutes: 180
+      runs_on_labels: '["ubuntu-latest"]'
diff --git a/test/deploy-tests-manifest.json b/test/deploy-tests-manifest.json
@@ -6,6 +6,83 @@
       "test/e2e/**/*.test.{t,j}s{,x}",
       "test/production/**/*.test.{t,j}s{,x}"
     ],
-    "exclude": []
+    "exclude": [
+      "test/e2e/app-dir/app-client-cache/client-cache.experimental.test.ts",
+      "test/e2e/app-dir/app-routes/app-custom-route-base-path.test.ts",
+      "test/e2e/app-dir/missing-suspense-with-csr-bailout/missing-suspense-with-csr-bailout.test.ts",
+      "test/e2e/app-dir/navigation/navigation.test.ts",
+      "test/e2e/app-dir/scss/npm-import-nested/npm-import-nested.test.ts",
+      "test/e2e/app-dir/static-shell-debugging/static-shell-debugging.test.ts",
+      "test/e2e/edge-pages-support/edge-document.test.ts",
+      "test/e2e/esm-externals/esm-externals.test.ts",
+      "test/e2e/new-link-behavior/material-ui.test.ts",
+      "test/e2e/next-phase/index.test.ts",
+      "test/e2e/react-dnd-compile/react-dnd-compile.test.ts",
+      "test/e2e/app-dir/app-client-cache/client-cache.defaults.test.ts",
+      "test/e2e/app-dir/app-prefetch-false-loading/app-prefetch-false-loading.test.ts",
+      "test/e2e/app-dir/app-routes-client-component/app-routes-client-component.test.ts",
+      "test/e2e/app-dir/app-static/app-static.test.ts",
+      "test/e2e/app-dir/metadata/metadata.test.ts",
+      "test/e2e/app-dir/parallel-routes-catchall-children-slot/parallel-routes-catchall-children-slot.test.ts",
+      "test/e2e/app-dir/parallel-routes-root-slot/parallel-routes-root-slot.test.ts",
+      "test/e2e/app-dir/ppr-navigations/avoid-popstate-flash/avoid-popstate-flash.test.ts",
+      "test/e2e/app-dir/ppr-navigations/loading-tsx-no-partial-rendering/loading-tsx-no-partial-rendering.test.ts",
+      "test/e2e/app-dir/ppr-navigations/stale-prefetch-entry/stale-prefetch-entry.test.ts",
+      "test/e2e/app-dir/ppr/ppr.test.ts",
+      "test/e2e/app-dir/revalidate-dynamic/revalidate-dynamic.test.ts",
+      "test/e2e/app-dir/scss/nm-module/nm-module.test.ts",
+      "test/e2e/app-dir/syntax-highlighter-crash/syntax-highlighter-crash.test.ts",
+      "test/e2e/app-dir/third-parties/basic.test.ts",
+      "test/e2e/new-link-behavior/stitches.test.ts",
+      "test/e2e/next-image-forward-ref/index.test.ts",
+      "test/e2e/app-dir/actions/app-action.test.ts",
+      "test/e2e/app-dir/conflicting-page-segments/conflicting-page-segments.test.ts",
+      "test/e2e/app-dir/i18n-hybrid/i18n-hybrid.test.js",
+      "test/e2e/app-dir/layout-params/layout-params.test.ts",
+      "test/e2e/app-dir/metadata-dynamic-routes/index.test.ts",
+      "test/e2e/app-dir/next-dynamic-css/next-dynamic-css.test.ts",
+      "test/e2e/app-dir/parallel-route-not-found-params/parallel-route-not-found-params.test.ts",
+      "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts",
+      "test/e2e/app-dir/rsc-webpack-loader/rsc-webpack-loader.test.ts",
+      "test/e2e/cancel-request/stream-cancel.test.ts",
+      "test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts",
+      "test/e2e/i18n-data-route/i18n-data-route.test.ts",
+      "test/e2e/react-compiler/react-compiler.test.ts",
+      "test/e2e/skip-trailing-slash-redirect/index.test.ts",
+      "test/e2e/tsconfig-module-preserve/index.test.ts",
+      "test/e2e/app-dir/actions/app-action-form-state.test.ts",
+      "test/e2e/app-dir/app-routes/app-custom-routes.test.ts",
+      "test/e2e/app-dir/app/useReportWebVitals.test.ts",
+      "test/e2e/app-dir/next-image/next-image-proxy.test.ts",
+      "test/e2e/app-dir/parallel-routes-revalidation/parallel-routes-revalidation.test.ts",
+      "test/e2e/app-dir/scss/3rd-party-module/3rd-party-module.test.ts",
+      "test/e2e/app-dir/scss/compilation-and-prefixing/compilation-and-prefixing.test.ts",
+      "test/e2e/app-dir/scss/nm-module-nested/nm-module-nested.test.ts",
+      "test/e2e/app-dir/scss/scss-mixins/scss-mixins.test.ts",
+      "test/e2e/app-dir/typeof-window/typeof-window.test.ts",
+      "test/e2e/edge-compiler-module-exports-preference/index.test.ts",
+      "test/e2e/socket-io/index.test.js",
+      "test/e2e/third-parties/index.test.ts",
+      "test/e2e/app-dir/actions-navigation/index.test.ts",
+      "test/e2e/app-dir/app-compilation/index.test.ts",
+      "test/e2e/app-dir/app-static/app-static-custom-handler.test.ts",
+      "test/e2e/app-dir/crypto-globally-available/crypto-globally-available.test.ts",
+      "test/e2e/app-dir/mdx/mdx.test.ts",
+      "test/e2e/app-dir/modularizeimports/modularizeimports.test.ts",
+      "test/e2e/app-dir/next-after-app/index.test.ts",
+      "test/e2e/app-dir/options-request/options-request.test.ts",
+      "test/e2e/app-dir/parallel-routes-layouts/parallel-routes-layouts.test.ts",
+      "test/e2e/app-dir/ppr-incremental/ppr-incremental.test.ts",
+      "test/e2e/app-dir/prefetch-searchparam/prefetch-searchparam.test.ts",
+      "test/e2e/app-dir/rsc-basic/rsc-basic.test.ts",
+      "test/e2e/app-dir/server-components-externals/index.test.ts",
+      "test/e2e/app-dir/webpack-loader-conditions/webpack-loader-conditions.test.ts",
+      "test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts",
+      "test/e2e/basepath.test.ts",
+      "test/e2e/next-test/next-test.test.ts",
+      "test/e2e/revalidate-reason/revalidate-reason.test.ts",
+      "test/e2e/swc-warnings/index.test.ts",
+      "test/e2e/type-module-interop/index.test.ts"
+    ]
   }
 }