From db74ba16e15736f1e284157f0f296d904a934f7c Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 30 Sep 2024 13:09:34 +0200 Subject: [PATCH 1/5] devops: unite channel tests into single GHA matrix workflows --- .github/workflows/tests_secondary.yml | 317 ++------------------------ 1 file changed, 17 insertions(+), 300 deletions(-) diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 0105f8c4bb8a5..74b41ac7a846b 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -184,56 +184,27 @@ jobs: PWTEST_TRACE: 1 PWTEST_CHANNEL: ${{ matrix.channel }} - chrome_stable_linux: - name: "Chrome Stable (Linux)" + test_chromium_channels: + name: Test ${{ matrix.channel }} on ${{ matrix.runs-on }} environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chrome - command: npm run ctest - bot-name: "chrome-stable-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: chrome - - chrome_stable_win: - name: "Chrome Stable (Win)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chrome - command: npm run ctest - bot-name: "chrome-stable-windows" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: chrome - - chrome_stable_mac: - name: "Chrome Stable (Mac)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: macos-latest + runs-on: ${{ matrix.runs-on }} + strategy: + fail-fast: false + matrix: + channel: [chrome, chrome-beta, msedge, msedge-beta, msedge-dev] + runs-on: [ubuntu-20.04, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/run-test with: - browsers-to-install: chrome + browsers-to-install: ${{ matrix.channel }} command: npm run ctest - bot-name: "chrome-stable-mac" + bot-name: chromium-channel-${{ matrix.channel }}-${{ matrix.runs-on }} flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} env: - PWTEST_CHANNEL: chrome + PWTEST_CHANNEL: ${{ matrix.channel }} chromium_tot: name: Chromium tip-of-tree ${{ matrix.os }} @@ -243,12 +214,13 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, macos-13, windows-latest] + headless: [true, false] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/run-test with: browsers-to-install: chromium-tip-of-tree - command: npm run ctest + command: npm run ctest ${{ matrix.headless && '--headless' || '' }} bot-name: "tip-of-tree-${{ matrix.os }}" flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} @@ -256,282 +228,27 @@ jobs: env: PWTEST_CHANNEL: chromium-tip-of-tree - chromium_tot_headed: - name: Chromium tip-of-tree headed ${{ matrix.os }} + firefox_beta: + name: Firefox Beta ${{ matrix.os }} environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chromium-tip-of-tree - command: npm run ctest -- --headed - bot-name: "tip-of-tree-headed-${{ matrix.os }}" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: chromium-tip-of-tree - - firefox_beta_linux: - name: "Firefox Beta (Linux)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: firefox-beta chromium - command: npm run ftest - bot-name: "firefox-beta-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: firefox-beta - - firefox_beta_win: - name: "Firefox Beta (Win)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: firefox-beta chromium - command: npm run ftest -- --workers=1 - bot-name: "firefox-beta-windows" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: firefox-beta - - firefox_beta_mac: - name: "Firefox Beta (Mac)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: macos-latest + os: [ubuntu-20.04, windows-latest, macos-latest] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/run-test with: browsers-to-install: firefox-beta chromium command: npm run ftest - bot-name: "firefox-beta-mac" + bot-name: "firefox-beta-${{ matrix.os }}" flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} env: PWTEST_CHANNEL: firefox-beta - edge_stable_mac: - name: "Edge Stable (Mac)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge - command: npm run ctest - bot-name: "edge-stable-mac" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge - - edge_stable_win: - name: "Edge Stable (Win)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge - command: npm run ctest - bot-name: "edge-stable-windows" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge - - edge_stable_linux: - name: "Edge Stable (Linux)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge - command: npm run ctest - bot-name: "edge-stable-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge - - edge_beta_mac: - name: "Edge Beta (Mac)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge-beta - command: npm run ctest - bot-name: "edge-beta-mac" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge-beta - - edge_beta_win: - name: "Edge Beta (Win)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge-beta - command: npm run ctest - bot-name: "edge-beta-windows" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge-beta - - edge_beta_linux: - name: "Edge Beta (Linux)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge-beta - command: npm run ctest - bot-name: "edge-beta-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge-beta - - edge_dev_mac: - name: "Edge Dev (Mac)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge-dev - command: npm run ctest - bot-name: "edge-dev-mac" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge-dev - - edge_dev_win: - name: "Edge Dev (Win)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge-dev - command: npm run ctest - bot-name: "edge-dev-windows" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge-dev - - edge_dev_linux: - name: "Edge Dev (Linux)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: msedge-dev - command: npm run ctest - bot-name: "edge-dev-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: msedge-dev - - chrome_beta_linux: - name: "Chrome Beta (Linux)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chrome-beta - command: npm run ctest - bot-name: "chrome-beta-linux" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: chrome-beta - - chrome_beta_win: - name: "Chrome Beta (Win)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chrome-beta - command: npm run ctest - bot-name: "chrome-beta-windows" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: chrome-beta - - chrome_beta_mac: - name: "Chrome Beta (Mac)" - environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} - runs-on: macos-latest - steps: - - uses: actions/checkout@v4 - - uses: ./.github/actions/run-test - with: - browsers-to-install: chrome-beta - command: npm run ctest - bot-name: "chrome-beta-mac" - flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} - flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} - flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} - env: - PWTEST_CHANNEL: chrome-beta - build-playwright-driver: name: "build-playwright-driver" runs-on: ubuntu-24.04 From b3ded115faa1433b17fc07b2ab42ef8de990d054 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 30 Sep 2024 13:26:15 +0200 Subject: [PATCH 2/5] Apply suggestions from code review Co-authored-by: Dmitry Gozman Signed-off-by: Max Schmitt --- .github/workflows/tests_secondary.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 74b41ac7a846b..920b92b653dac 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -214,14 +214,14 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, macos-13, windows-latest] - headless: [true, false] + headed: ['--headed', ''] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/run-test with: browsers-to-install: chromium-tip-of-tree - command: npm run ctest ${{ matrix.headless && '--headless' || '' }} - bot-name: "tip-of-tree-${{ matrix.os }}" + command: npm run ctest -- ${{ matrix.headed }} + bot-name: "tip-of-tree-${{ matrix.os }}${{ matrix.headed }}" flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} From aa5204e10c00a057c81a744224a228c401cd22f5 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 30 Sep 2024 13:27:15 +0200 Subject: [PATCH 3/5] Update .github/workflows/tests_secondary.yml Co-authored-by: Dmitry Gozman Signed-off-by: Max Schmitt --- .github/workflows/tests_secondary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 920b92b653dac..01f0f9a86b2c2 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -207,7 +207,7 @@ jobs: PWTEST_CHANNEL: ${{ matrix.channel }} chromium_tot: - name: Chromium tip-of-tree ${{ matrix.os }} + name: Chromium tip-of-tree ${{ matrix.os }}${{ matrix.headed }} environment: ${{ github.event_name == 'push' && 'allow-uploading-flakiness-results' || null }} runs-on: ${{ matrix.os }} strategy: From 74189a8a2a3294abacc09158db9d8a76a4d87fb0 Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 30 Sep 2024 13:27:21 +0200 Subject: [PATCH 4/5] Update .github/workflows/tests_secondary.yml Co-authored-by: Dmitry Gozman Signed-off-by: Max Schmitt --- .github/workflows/tests_secondary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 01f0f9a86b2c2..1bcdce78f935c 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -199,7 +199,7 @@ jobs: with: browsers-to-install: ${{ matrix.channel }} command: npm run ctest - bot-name: chromium-channel-${{ matrix.channel }}-${{ matrix.runs-on }} + bot-name: ${{ matrix.channel }}-${{ matrix.runs-on }} flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }} From 6ab6de309188073c5a8648080515544d493446ee Mon Sep 17 00:00:00 2001 From: Max Schmitt Date: Mon, 30 Sep 2024 13:31:00 +0200 Subject: [PATCH 5/5] nit --- .github/workflows/tests_secondary.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_secondary.yml b/.github/workflows/tests_secondary.yml index 1bcdce78f935c..aa9ffa8501c55 100644 --- a/.github/workflows/tests_secondary.yml +++ b/.github/workflows/tests_secondary.yml @@ -221,7 +221,7 @@ jobs: with: browsers-to-install: chromium-tip-of-tree command: npm run ctest -- ${{ matrix.headed }} - bot-name: "tip-of-tree-${{ matrix.os }}${{ matrix.headed }}" + bot-name: "chromium-tip-of-tree-${{ matrix.os }}${{ matrix.headed }}" flakiness-client-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_CLIENT_ID }} flakiness-tenant-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_TENANT_ID }} flakiness-subscription-id: ${{ secrets.AZURE_FLAKINESS_DASHBOARD_SUBSCRIPTION_ID }}