ci: visual tests (#2534) (#2566)

* ci: testing visual comp

* ci: more test

* test: changes

* chore: update CI

* chore: fix npm

* save

* chore: working locally

* chore: working logic

* chore: save

* chore: testing CI

* ci: test ci

* ci: more test

* ci: more ci tests

* ci: test

* chore: updated test

* chore: test

* more tests

* ci: test

* ci: change folder structure

* chore: fixed ci

* chore: reduce test feedback

* chore: more test

* ci: path change

* ci: more test

* ci: update tar config

* ci: retrest

* chore: test

* chore: test

* ci: more test

* chore: more test

* save

* ci: different workflow

* chore: fixed logic

* chore: more tests

* core: ready for PR test

* ci: added missing steps

* ci: screenshot logic

* ci: trigger

* ci: more logic

* ci: fix test path

* test: new logic

* ci: fix download path

* chore: ready

* save

* ci: more tests

* ci: fix screenshots

* ci: more changes

* chore: more tests

* ci: more tests

* chore: restart

* chore: update

* ci: removed Appzi from builds

* chore: save

* ci: new test

* save

* ci: updated artifacts logic

* chore: verify self-hosted runner

* chore: ready for merge

(cherry picked from commit 0670689)

Co-authored-by: Karl Cardenas <karl@spectrocloud.com>

.dockerignore

@@ -4,4 +4,8 @@ CHANGES
 .gitignore
 .cache
 prow/
-.env
+.env
+screenshots/
+tests/screenshot.spec.ts-snapshots/
+tests-results/
+playwright-report/

.github/workflows/clean-up-report.yaml

@@ -0,0 +1,61 @@
+name: Delete Visual Tests Reports
+
+on:
+  delete:
+    branches-ignore: [master, main, gh-pages]
+
+
+concurrency:
+  group: ${{ github.event.ref }}
+  cancel-in-progress: true
+
+jobs:
+  delete_reports:
+    name: Delete Reports
+    runs-on: ubuntu-latest
+    env:
+      # Contains all reports for deleted branch
+      BRANCH_REPORTS_DIR: reports/${{ github.event.ref }}
+    steps:
+      - name: Checkout GitHub Pages Branch
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+
+      - name: Set Git User
+        # see: https://github.com/actions/checkout/issues/13#issuecomment-724415212
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Check for workflow reports
+        run: |
+          if [ -z "$(ls -A $BRANCH_REPORTS_DIR)" ]; then
+            echo "BRANCH_REPORTS_EXIST="false"" >> $GITHUB_ENV
+          else
+            echo "BRANCH_REPORTS_EXIST="true"" >> $GITHUB_ENV
+          fi
+          
+      - name: Delete reports from repo for branch
+        if: ${{ env.BRANCH_REPORTS_EXIST == 'true' }}
+        timeout-minutes: 3
+        run: |
+          cd $BRANCH_REPORTS_DIR/..
+
+          rm -rf ${{ github.event.ref }}
+          git add .
+          git commit -m "workflow: remove all reports for branch ${{ github.event.ref }}"
+
+          while true; do
+            git pull --rebase
+            if [ $? -ne 0 ]; then
+              echo "Failed to rebase. Please review manually."
+              exit 1
+            fi
+
+            git push
+            if [ $? -eq 0 ]; then
+              echo "Successfully pushed HTML reports to repo."
+              exit 0
+            fi
+          done

.github/workflows/screenshot_capture.yaml

@@ -0,0 +1,132 @@
+name: Screenshot Capture
+# This workflow is triggered when a workflow run of the "Release to Production" workflow is completed or when manually triggered.
+# The primary purpose of this workflow is to take screenshots of the website using Playwright and upload them as artifacts.
+# The screenshots can be used to compare the visual appearance of the website before and after a release.
+# The workflow Visual Comparison uses these screenshots to generate a visual comparison report.
+
+on:
+  workflow_run:
+    workflows: ["Release to Production"]
+    types: [completed]
+  workflow_dispatch:
+
+
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }}
+  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
+  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
+  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
+
+
+jobs:
+
+  create-assets:
+    name: asset-builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js Environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+          cache: "npm"
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium
+
+      - name: Build Website
+        run: make build
+
+      - name: Upload Build
+        uses: actions/upload-artifact@v4
+        with:
+          name: "build"
+          path: |
+            build/
+          if-no-files-found: error
+          retention-days: 1
+
+
+
+
+  visual-snapshots:
+    runs-on: ubuntu-latest
+    needs: [create-assets]
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js Environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+          cache: "npm"
+
+      - name: Install Dependencies
+        uses: Wandalen/wretry.action@v3
+        with: 
+            command: npm ci
+            attempt_limit: 3
+            attempt_delay: 60000 # 1 minute
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium
+
+      - name: retry action
+        uses: Wandalen/wretry.action@v3
+        with: 
+          attempt_limit: 3
+          action: actions/download-artifact@v4
+          with: |
+            name: build
+            path: build
+          attempt_delay: 60000 # 1 minute
+
+      - name: Take Screenshots with Playwright
+        run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0
+
+      - name: Upload Screenshots
+        uses: actions/upload-artifact@v4
+        id: screenshots
+        with:
+          name: screenshots-${{ matrix.shardIndex }}
+          path: |
+            screenshots/
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-snapshots:
+    name: Merge Screenshots
+    runs-on: ubuntu-latest
+    needs: [visual-snapshots]
+    steps:
+
+    - name: Download blob reports from GitHub Actions Artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: screenshots
+        pattern: screenshots-*
+        merge-multiple: true
+
+    - name: Upload Screenshots
+      uses: actions/upload-artifact@v4
+      id: screenshots
+      with:
+        name: "screenshots"
+        path: |
+          screenshots/
+        if-no-files-found: error
+        retention-days: 3