meta: added chromatic for visual regression tests (#5583)

* meta: added chromatic for visual regression tests

* chore: remove snapshots

* chore: remove storybook test runner and snapshot tests

* chore: added jest-junit reporter

* chore: remove unrelated change

* chore: remove snapshots

* chore: build storybook

* chore: add chromatic with secret token

* chore: no need of chromatic package anymore

* chore: no --ci argument

* chore: removed old chromatic command

* chore: updated environment name

Signed-off-by: Claudio Wunder <cwunder@gnome.org>

* chore: updated conditions for dependabot PRs and Merge Queues

---------

Signed-off-by: Claudio Wunder <cwunder@gnome.org>

.github/workflows/build.yml

@@ -105,21 +105,26 @@ jobs:
           NEXT_TELEMETRY_DISABLED: 1
 
       - name: Analyse Build
-        if: github.event_name == 'pull_request'
+        # We don't need build analysis for Dependabot PRs and also when the GitHub Event is not a Pull Request
+        # i.e. if the Event is a Merge Queue Event
+        if: startsWith(github.event.pull_request.head.ref, 'dependabot/') == false && github.event_name == 'pull_request'
         # We generate a Bundle Analysis Report
         # See https://github.com/hashicorp/nextjs-bundle-analysis
         run: npx --package=nextjs-bundle-analysis@0.5.0 report
 
       - name: Upload Build Analysis
-        if: github.event_name == 'pull_request'
+        # This prevents this step from running if "Analyse Build" got cancelled; Which gets cancelled if
+        # the curruent branch comes from Dependabot or the Event is not a Pull Request (i.e. Merge Queue Event)
+        if: success()
         # We upload the Bundle Analysis Artifact so it can be used on another Workflow
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
         with:
           name: bundle-analysis
           path: .next/analyze/__bundle_analysis.json
 
       - name: Save Build Cache
-        if: github.event_name == 'pull_request'
+        # This prevents this step from running if Upload Build Analysis" got cancelled
+        if: success()
         uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
         with:
           path: |

.github/workflows/pull-request-target.yml

@@ -47,14 +47,13 @@ jobs:
       - name: Wait for Tests to Finish
         uses: lewagon/wait-on-check-action@e106e5c43e8ca1edea6383a39a01c5ca495fd812
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.ref }}
           check-name: 'Tests'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           wait-interval: 30
 
       - name: Download PR Coverage Report
         uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
-        if: success() && github.event.number
         with:
           name: coverage-report
           workflow: pull-request.yml
@@ -70,6 +69,49 @@ jobs:
           junitxml-path: ./junit.xml
           junitxml-title: Unit Test Report
 
+  visual-regression:
+    if: startsWith(github.event.pull_request.head.ref, 'dependabot/') == false
+    name: Visual Regression Upload
+    runs-on: ubuntu-latest
+    environment:
+      name: Storybook
+      url: ${{ steps.chromatic-deploy.outputs.storybookUrl }}
+
+    steps:
+      - name: Git Checkout (`base_ref`)
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+        with:
+          # We only need to fetch the latest commit of the `base_ref` here for the analysis steps
+          fetch-depth: 1
+          # For this step we don't need to checkout on the userland codebase
+          # As we can simply checkout on the latest commit of the `base_ref` for the analysis steps
+          ref: ${{ github.event.pull_request.base.ref }}
+
+      - name: Wait for Tests to Finish
+        uses: lewagon/wait-on-check-action@e106e5c43e8ca1edea6383a39a01c5ca495fd812
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          check-name: 'Tests'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 30
+
+      - name: Download PR Storybook Build
+        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615
+        with:
+          name: storybook-build
+          workflow: pull-request.yml
+          branch: ${{ github.event.pull_request.head.ref }}
+          if_no_artifact_found: fail
+
+      - name: Start Visual Regression Tests (Chromatic)
+        id: chromatic-deploy
+        uses: chromaui/action@v1
+        with:
+          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+          storybookBuildDir: storybook-static
+          exitOnceUploaded: true
+          onlyChanged: true
+
   analysis-report:
     # We want to skip our Pull Request Comments on Dependabot PRs
     if: startsWith(github.event.pull_request.head.ref, 'dependabot/') == false
@@ -89,15 +131,15 @@ jobs:
       - name: Wait for Linux Build to Finish
         uses: lewagon/wait-on-check-action@e106e5c43e8ca1edea6383a39a01c5ca495fd812
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.ref }}
           check-name: 'Build on ubuntu-latest'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           wait-interval: 30
 
       - name: Wait for Windows Build to Finish
         uses: lewagon/wait-on-check-action@e106e5c43e8ca1edea6383a39a01c5ca495fd812
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.ref }}
           check-name: 'Build on windows-latest'
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           wait-interval: 30

.github/workflows/pull-request.yml

@@ -107,7 +107,9 @@ jobs:
         run: npx --package=turbo@latest -- turbo prettier ${{ needs.base.outputs.turbo_args }}
 
       - name: Save Lint Cache
-        if: github.event_name == 'pull_request'
+        # We don't need to upload a Lint Cache for Dependabot PRs and also when the GitHub Event is not a Pull Request
+        # i.e. if the Event is a Merge Queue Event
+        if: startsWith(github.event.pull_request.head.ref, 'dependabot/') == false && github.event_name == 'pull_request'
         uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
         with:
           path: |
@@ -157,7 +159,9 @@ jobs:
         run: npx --package=turbo@latest -- turbo test:unit ${{ needs.base.outputs.turbo_args }} -- --ci --coverage
 
       - name: Upload Coverage Report
-        if: github.event_name == 'pull_request'
+        # We don't need coverage report for Dependabot PRs and also when the GitHub Event is not a Pull Request
+        # i.e. if the Event is a Merge Queue Event
+        if: startsWith(github.event.pull_request.head.ref, 'dependabot/') == false && github.event_name == 'pull_request'
         # We upload the Coverage Report so it can be used on another Workflow
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
         with:
@@ -166,13 +170,29 @@ jobs:
             coverage/
             junit.xml
 
-      - name: Run Storybook Tests
+      - name: Build Storybook
+        # This prevents this step from running if "Upload Coverage Report" got cancelled; Which gets cancelled if
+        # the curruent branch comes from Dependabot or the Event is not a Pull Request (i.e. Merge Queue Event)
+        if: success()
+        # We Build Storybook Locally and then upload it so it can be used on another Workflow
         # We want to enforce that the actual `turbo@latest` package is used instead of a possible hijack from the user
         # the `${{ needs.base.outputs.turbo_args }}` is a string substitution happening from the base job
-        run: npx --package=turbo@latest -- turbo test:storybook:local ${{ needs.base.outputs.turbo_args }} -- --ci
+        run: npx --package=turbo@latest -- turbo storybook:build ${{ needs.base.outputs.turbo_args }}
+
+      - name: Upload Storybook Build
+        # This prevents this step from running if "Build Storybook" got cancelled; Which gets cancelled if
+        # the curruent branch comes from Dependabot or the Event is not a Pull Request (i.e. Merge Queue Event)
+        if: success()
+        # We upload the Storybook Build so it can be used on another Workflow
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce
+        with:
+          name: storybook-build
+          path: storybook-static
 
       - name: Save Tests Cache
-        if: github.event_name == 'pull_request'
+        # This prevents this step from running if "Upload Storybook Build" got cancelled; Which gets cancelled if
+        # the curruent branch comes from Dependabot or the Event is not a Pull Request (i.e. Merge Queue Event)
+        if: success()
         uses: actions/cache/save@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8
         with:
           path: |

CONTRIBUTING.md

@@ -147,10 +147,6 @@ This repository contains several scripts and commands for performing numerous ta
 - `npx turbo storybook:build` builds Storybook as a static web application for publishing
 - `npx turbo test` runs all tests locally
   - `npx turbo test:unit` runs jest (unit-tests) locally
-  - `npx turbo test:storybook` runs storybook test-runner tests
-    - `npx turbo test:storybook:local` runs storybook test-runner tests with local storybook
-    - `npx turbo test:storybook:watch` keeps running storybook test-runner locally on every story change
-    - `npx turbo test:storybook:snapshot` generates and updates snapshots for all storybook components.
 
 </details>
 

README.md

@@ -108,6 +108,13 @@ The current integration is owned by the OpenJS Foundation and managed by the Web
 
 [Dependency Pinning][]
 
+## Thanks
+
+- Thanks to all contributors and collaborators that make this project possible.
+- Thanks to [Chromatic](https://www.chromatic.com/) for providing the visual testing platform that helps us review UI changes and catch visual regressions.
+- Thanks to [Vercel](https://www.vercel.com/) for providing the infrastructure that serves and powers the Node.js Website
+- Thanks to [Cloudflare](https://cloudflare.com) for providing the infrastructure that serves Node.js's Website, Node.js's CDN and more.
+
 [code of conduct]: https://github.com/nodejs/admin/blob/main/CODE_OF_CONDUCT.md
 [contribution guidelines]: ./CONTRIBUTING.md
 [content vs code]: ./CONTENT_VS_CODE.md