chore(ci): shard e2e components tests

.github/workflows/e2e-ct.yml

@@ -6,12 +6,82 @@ on:
   push:
     branches: [next]
 jobs:
-  test:
+  install:
     timeout-minutes: 30
     runs-on: ubuntu-latest
     env:
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
+        project: [chromium, firefox, webkit]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}-
+            v1-${{ runner.os }}-pnpm-store-
+            v1-${{ runner.os }}-
+
+      - name: Install project dependencies
+        run: pnpm install
+
+      - name: Build packages
+        run: pnpm build --output-logs=full --log-order=grouped
+
+      # Caches build from either PR or next
+      - name: Cache build
+        id: cache-e2e-build
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-e2e-build
+        with:
+          path: |
+            ./*
+            !**/node_modules/**
+          # Unique key for a workflow run. Should be invalidated in the next run
+          key: ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }}
+
+  playwright-ct-test:
+    timeout-minutes: 30
+    needs: [install]
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
+        project: [chromium, firefox, webkit]
+        # Add more shards here if needed
+        shardIndex: [1, 2]
+        shardTotal: [2]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -58,38 +128,153 @@ jobs:
           path: ~/.cache/ms-playwright
           key: playwright-browsers-${{ env.PLAYWRIGHT_VERSION }}
       - name: Install Playwright Browsers
+        # TODO: Fix webkit caching when downloading from cache
+        # for some reason it doesn't work without installing again
+        # if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
         run: npx playwright install --with-deps
-      - name: Build packages
-        run: pnpm build --output-logs=full --log-order=grouped
+
+      - name: Restore build cache
+        uses: actions/cache/restore@v4
+        id: restore-build
+        env:
+          cache-name: cache-e2e-build
+        with:
+          path: |
+            ./*
+            !**/node_modules/**
+          key: ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }}
+          # If the cached build from the pervious step is not available. Fail the build
+          fail-on-cache-miss: true
+
       - name: Run end-to-end tests
-        run: pnpm --filter sanity test:ct
         env:
+          PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }}
           NODE_OPTIONS: --max_old_space_size=8192
+        run: pnpm --filter sanity test:ct --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: playwright-ct-report
+          path: ${{ github.workspace }}/packages/sanity/blob-report
+          retention-days: 30
+
+  merge-reports:
+    if: always()
+    needs: [playwright-ct-test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - uses: pnpm/action-setup@v4
+        name: Install pnpm
+        id: pnpm-install
+        with:
+          run_install: false
+
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        shell: bash
+        run: |
+          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
+
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ env.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            v1-${{ runner.os }}-pnpm-store-${{ env.cache-name }}-
+            v1-${{ runner.os }}-pnpm-store-
+            v1-${{ runner.os }}-
+
+      - name: Install project dependencies
+        run: pnpm install
+
+      - name: Download blob reports from Github Actions Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: playwright-ct-report
+          path: ${{ github.workspace }/packages/sanity/playwright-ct/playwright-ct-report
+
+      - name: Merge into HTML Report
+        run: npx playwright merge-reports --reporter json ${{ github.workspace }/packages/sanity/playwright-ct/playwright-ct-report >> ${{ github.workspace }/packages/sanity/playwright-ct/playwright-ct-report/playwright-ct-test-results.json
+
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v3
+        with:
+          name: html-report--attempt-${{ github.run_attempt }}
+          path: playwright-ct-report
+          retention-days: 30
+
       - name: Get Current Job Log URL
         uses: Tiryoh/gha-jobid-action@c1d1cf7334b70c29374ae382b91053674c8049a2 # v1
         id: job_html_url
         if: always()
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          job_name: "test"
+          # Get the first run's ID even though it might be correct it should bring to the right place
+          job_name: "playwright-ct-test (chromium, 1, 2)"
+
       - name: Parse playwright report
         id: parse-playwright-report
         if: always()
         run: pnpm --filter sanity write:playwright-report-as-pr-comment
         env:
           GITHUB_WORKFLOW_URL: "${{ steps.job_html_url.outputs.html_url }}"
+          REPORT_JSON_PATH: "${{ github.workspace }}/playwright-ct-report/playwright-ct-test-results.json"
+
       - name: PR comment with report
         uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2
         if: ${{ always() && steps.parse-playwright-report.conclusion == 'success' && github.event_name == 'pull_request' }}
         with:
           comment_tag: "playwright-ct-report"
-          filePath: ${{ github.workspace }}/packages/sanity/playwright-ct/results/playwright-report-pr-comment.md
+          filePath: ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report/playwright-report-pr-comment.md
 
       - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: playwright-ct-report
           path: |
             ${{ github.workspace }}/packages/sanity/playwright-ct/report
+            ${{ github.workspace }}/packages/sanity/playwright-ct/playwright-ct-report
             ${{ github.workspace }}/packages/sanity/playwright-ct/results
           retention-days: 30
+
+  cleanup:
+    permissions:
+      contents: read
+      actions: write # needed to delete the cache
+    timeout-minutes: 30
+    name: Cleanup (${{ matrix.project }})
+    runs-on: ubuntu-latest
+    needs: [playwright-ct-test]
+
+    strategy:
+      # we want to know if a test fails on a specific node version
+      fail-fast: false
+      matrix:
+        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
+        project: [chromium, firefox, webkit]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      # Delete the cache so it is only used once
+      - name: Delete Cache
+        run: gh cache delete ${{ runner.os }}-${{ matrix.project }}-${{ env.cache-name }}-${{ github.run_id }}
+        env:
+          cache-name: cache-e2e-build
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

packages/sanity/.gitignore

@@ -26,6 +26,7 @@
 /playwright-ct/report
 /playwright-ct/results
 /playwright-ct/template/.cache/
+/playwright-ct/playwright-ct-report
 
 # Deleted legacy exports
 /migrate.js

packages/sanity/playwright-ct.config.ts

@@ -25,16 +25,18 @@ export default defineConfig({
   /* Opt out of parallel tests on CI. */
   workers: process.env.CI ? 1 : undefined,
   /* Reporter to use. See https://playwright.dev/docs/test-reporters */
-  reporter: [
-    ['list'],
-    ['html', {outputFolder: HTML_REPORT_PATH}],
-    [
-      'json',
-      {
-        outputFile: path.join(ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json'),
-      },
-    ],
-  ],
+  reporter: isCI
+    ? [['list'], ['blob']]
+    : [
+        ['list'],
+        ['html', {outputFolder: HTML_REPORT_PATH}],
+        [
+          'json',
+          {
+            outputFile: path.join(ARTIFACT_OUTPUT_PATH, 'playwright-ct-test-results.json'),
+          },
+        ],
+      ],
 
   /* Maximum time one test can run for. */
   timeout: 30 * 1000,

packages/sanity/playwright-ct/scripts/parsePlaywrightReportJson.ts

@@ -7,7 +7,9 @@ import {inspect} from 'node:util'
 import {generateOutput, groupTests, readJsonFile} from './utils'
 
 const DEBUG = Boolean(parseInt(process.env.DEBUG || '0', 2))
-const DEFAULT_ARTIFACT_OUTPUT_PATH = path.resolve(path.join(__dirname, '..', 'results'))
+const DEFAULT_ARTIFACT_OUTPUT_PATH = path.resolve(
+  path.join(__dirname, '..', 'playwright-ct-report'),
+)
 
 /**
  * Summarize the Playwright report JSON into a Markdown string that can be posted as a comment to PRs