E2E Tests #1097
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Tests | |
| on: | |
| # See: https://crontab.guru/examples.html | |
| # Run at 6 A.M and 6 P.M. (CST) | |
| schedule: | |
| - cron: '0 12,0 * * *' | |
| workflow_dispatch: | |
| workflow_call: | |
| jobs: | |
| versions: | |
| name: Get Versions 📜 | |
| uses: CrisisCleanup/crisiscleanup-4-web/.github/workflows/get-versions.yml@master | |
| e2e: | |
| name: 🧪 E2E Tests 💻 | ${{ matrix.stage }} | ${{ matrix.project }} | |
| timeout-minutes: 60 | |
| runs-on: ${{ matrix.os }} | |
| needs: [versions] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| stage: [development, staging, production] | |
| project: [chromium, firefox, webkit] | |
| include: | |
| - project: chromium | |
| os: ubuntu-latest | |
| cache_dir: ~/.cache/ms-playwright | |
| - project: firefox | |
| os: ubuntu-latest | |
| cache_dir: ~/.cache/ms-playwright | |
| - project: webkit | |
| os: macos-12 | |
| cache_dir: ~/Library/Caches/ms-playwright | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v2 | |
| with: | |
| version: ${{ needs.versions.outputs.PNPM_VERSION }} | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ needs.versions.outputs.NODE_VERSION }} | |
| cache: pnpm | |
| # NOTE: Using single quotes when setting $GITHUB_ENV in-case secret has special characters | |
| - name: 🔧 Setup Common Env | |
| run: | | |
| echo 'PW_SKIP_WEBSERVER=1' >> $GITHUB_ENV | |
| - name: 🔧 Setup Development Test Env | |
| if: matrix.stage == 'development' | |
| run: | | |
| echo 'PW_TEST_TITLE_PREFIX=DEV: ' >> $GITHUB_ENV | |
| echo 'VITE_APP_BASE_URL=https://app.dev.crisiscleanup.io' >> $GITHUB_ENV | |
| echo 'TEST_APP_EMAIL=${{ secrets.TEST_APP_EMAIL_DEV }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_PASSWORD=${{ secrets.TEST_APP_PASSWORD_DEV }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_ADMIN_EMAIL=${{ secrets.TEST_APP_ADMIN_EMAIL_DEV }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_ADMIN_PASSWORD=${{ secrets.TEST_APP_ADMIN_PASSWORD_DEV }}' >> $GITHUB_ENV | |
| - name: 🔧 Setup Staging Test Env | |
| if: matrix.stage == 'staging' | |
| run: | | |
| echo 'PW_TEST_TITLE_PREFIX=STAGING: ' >> $GITHUB_ENV | |
| echo 'VITE_APP_BASE_URL=https://app.staging.crisiscleanup.io' >> $GITHUB_ENV | |
| echo 'TEST_APP_EMAIL=${{ secrets.TEST_APP_EMAIL_PROD }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_PASSWORD=${{ secrets.TEST_APP_PASSWORD_PROD }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_ADMIN_EMAIL=${{ secrets.TEST_APP_ADMIN_EMAIL_PROD }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_ADMIN_PASSWORD=${{ secrets.TEST_APP_ADMIN_PASSWORD_PROD }}' >> $GITHUB_ENV | |
| - name: 🔧 Setup Production Test Env | |
| if: matrix.stage == 'production' | |
| run: | | |
| # add 'PROD: ' prefix in test titles. ex: 'PROD: should do this - ( @production )' | |
| echo 'PW_TEST_TITLE_PREFIX=PROD: ' >> $GITHUB_ENV | |
| echo 'VITE_APP_BASE_URL=https://crisiscleanup.org' >> $GITHUB_ENV | |
| echo 'TEST_APP_EMAIL=${{ secrets.TEST_APP_EMAIL_PROD }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_PASSWORD=${{ secrets.TEST_APP_PASSWORD_PROD }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_ADMIN_EMAIL=${{ secrets.TEST_APP_ADMIN_EMAIL_PROD }}' >> $GITHUB_ENV | |
| echo 'TEST_APP_ADMIN_PASSWORD=${{ secrets.TEST_APP_ADMIN_PASSWORD_PROD }}' >> $GITHUB_ENV | |
| - name: Install dependencies | |
| run: pnpm install | |
| - name: 🔧 Set Playwright Version | |
| run: pnpm exec playwright --version > .github/.playwright-version | |
| - name: ⚡️ Cache playwright binaries | |
| uses: actions/cache@v4 | |
| id: playwright-cache | |
| with: | |
| path: ${{ matrix.cache_dir }} | |
| key: ${{ runner.os }}-${{ matrix.project }}-pw-${{ hashFiles('**/.playwright-version') }} | |
| - name: 📥 Install ${{ matrix.project }} with Playwright | |
| if: steps.playwright-cache.outputs.cache-hit != 'true' | |
| run: pnpm exec playwright install --with-deps ${{ matrix.project }} | |
| - name: 🎭 Playwright tests | |
| run: pnpm run test:e2e:${{ matrix.stage }} --project=${{ matrix.project }} | |
| - name: 📤 Upload blob report to GitHub Actions Artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: all-blob-reports-${{ matrix.stage }} | |
| path: blob-report | |
| retention-days: 1 | |
| - name: Save Test Result | |
| if: always() | |
| run: | | |
| echo ${{ job.status }} > test-result-${{ matrix.stage }}-${{ matrix.project }}.txt | |
| - name: Upload Test Result | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: matrix-outputs | |
| path: test-result-${{ matrix.stage }}-${{ matrix.project }}.txt | |
| aggregate-results: | |
| name: Aggregate Test Results | |
| needs: [e2e] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| outputs: | |
| overall_status: ${{ steps.aggregate.outputs.overall_status }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: matrix-outputs | |
| path: test-results | |
| - name: Aggregate Test Results | |
| id: aggregate | |
| run: | | |
| #!/bin/bash | |
| overall_status="success" | |
| # Loop through each test result file | |
| for file in test-results/*.txt; do | |
| if [[ -f "$file" ]]; then | |
| status=$(cat "$file") | |
| echo "Test result in $file: $status" | |
| if [[ "$status" == "failure" ]]; then | |
| overall_status="failure" | |
| break | |
| fi | |
| fi | |
| done | |
| # Output the overall status | |
| echo "Overall status: $overall_status" | |
| echo "overall_status=$overall_status" >> $GITHUB_OUTPUT | |
| notify-failed: | |
| name: 🔴 Notify Failure 🔴 | |
| needs: [e2e, aggregate-results] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Notify Slack | |
| uses: rtCamp/action-slack-notify@v2 | |
| if: ${{ needs.aggregate-results.outputs.overall_status == 'failure' }} | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| SLACK_ICON: ${{ secrets.SLACK_ICON }} | |
| SLACK_USERNAME: 'CrisisCleanup Bot' | |
| SLACK_CHANNEL: 'qa' | |
| SLACK_COLOR: '#D94F37' | |
| SLACK_TITLE: 'E2e Tests Failed!' | |
| SLACK_MESSAGE: '🧪 E2e Tests Failed... Someone broke something! 😦' | |
| notify-success: | |
| name: 🟢 Notify Success 🟢 | |
| needs: [e2e, aggregate-results] | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Notify Slack | |
| uses: rtCamp/action-slack-notify@v2 | |
| if: ${{ needs.aggregate-results.outputs.overall_status == 'success' }} | |
| env: | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| SLACK_ICON: ${{ secrets.SLACK_ICON }} | |
| SLACK_USERNAME: 'CrisisCleanup Bot' | |
| SLACK_CHANNEL: 'qa' | |
| SLACK_COLOR: '#13E768' | |
| SLACK_TITLE: 'E2e Tests Passed!' | |
| SLACK_MESSAGE: '🧪 E2e Tests Passed!!' | |
| merge-e2e-reports: | |
| # Merge reports after playwright-tests, even if some shards have failed | |
| name: 🔄 Merge Playwright Reports | ${{ matrix.stage }} | |
| if: always() | |
| needs: [versions, e2e] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| stage: [development, staging, production] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: pnpm/action-setup@v2 | |
| with: | |
| version: ${{ needs.versions.outputs.PNPM_VERSION }} | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: ${{ needs.versions.outputs.NODE_VERSION }} | |
| cache: pnpm | |
| - name: 📥 Install dependencies | |
| run: pnpm install | |
| - name: 📥 Download blob reports from GitHub Actions Artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: all-blob-reports-${{ matrix.stage }} | |
| path: all-blob-reports | |
| - name: 🔄 Merge all blob reports | |
| env: | |
| PLAYWRIGHT_JSON_OUTPUT_NAME: e2e-results.json | |
| run: pnpm exec playwright merge-reports --reporter html,json,github,markdown ./all-blob-reports | |
| - name: 📨 Send report to slack | |
| env: | |
| SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.SLACK_BOT_USER_OAUTH_TOKEN }} | |
| REPORT_NAME: Playwright Report | ${{ matrix.stage }} | |
| BUILD_ID: ${{ github.run_id }} | |
| BRANCH_NAME: ${{ github.ref_name }} | |
| COMMIT_ID: ${{ github.sha }} | |
| # Todo: replace with html report url when we upload html reports somewhere | |
| RESULT_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| cat > cli_config.json <<EOF | |
| { | |
| "sendResults": "always", | |
| "slackLogLevel": "debug", | |
| "sendUsingBot": { | |
| "channels": ["qa"] | |
| }, | |
| "showInThread": true, | |
| "meta": [ | |
| { "key": "name", "value" : "__ENV_REPORT_NAME"}, | |
| { "key": "build", "value" : "__ENV_BUILD_ID"}, | |
| { "key": "branch", "value" : "__ENV_BRANCH_NAME"}, | |
| { "key": "commit", "value" : "__ENV_COMMIT_ID"}, | |
| { "key": "results", "value" : "__ENV_RESULT_URL"} | |
| ], | |
| "maxNumberOfFailures": 10, | |
| "disableUnfurl": true | |
| } | |
| EOF | |
| echo "Created cli_config.json for playwright-slack-report..." | |
| cat cli_config.json | |
| echo "Playwright JSON report:" | |
| cat e2e-results.json | |
| echo "Sending report to slack..." | |
| pnpm exec playwright-slack-report --config=cli_config.json --json-results=e2e-results.json | |
| - name: 🛠️ Generate Job Summary | |
| run: | | |
| echo "# 🧪 E2E Summary 🧪" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| cat report.md >> $GITHUB_STEP_SUMMARY # report.md is generated from markdown reporter | |
| - name: 📤 Upload HTML report (${{ matrix.stage }}) | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: e2e-report-${{ matrix.stage }}-${{ github.run_id }}-attempt_${{ github.run_attempt }} | |
| path: playwright-report | |
| retention-days: 7 |