smoke.yml

name: smoke

on:
  push:
    branches: [main]
  pull_request: # run on all PRs, not just PRs to a particular branch

env:
  PUPPETEER_SKIP_DOWNLOAD: 1

jobs:
  # `smoke` runs as a matrix across 6 jobs:
  #  * The smoke tests are split into 3 batches, to parallelize.
  #  * Then, those are run with both Chrome stable and ToT Chromium, in parallel
  smoke:
    strategy:
      matrix:
        chrome-channel: ['stable', 'ToT']
        smoke-test-shard: [1, 2, 3]
      # e.g. if set 1 fails, continue with set 2 anyway
      fail-fast: false
    runs-on: ubuntu-latest
    env:
      # The total number of shards. Set dynamically when length of *single* matrix variable is
      # computable. See https://git.luolix.topmunity/t/get-length-of-strategy-matrix-or-get-all-matrix-options/18342
      SHARD_TOTAL: 3
      FORCE_COLOR: true
    # Job named e.g. "Chrome stable 1/3".
    name: Chrome ${{ matrix.chrome-channel }} ${{ matrix.smoke-test-shard }}/3

    steps:
    - name: git clone
      uses: actions/checkout@v3
      with:
        # Depth of at least 2 for codecov coverage diffs. See https://github.com/GoogleChrome/lighthouse/pull/12079
        fetch-depth: 2

    - name: Use Node.js 18.x
      uses: actions/setup-node@v3
      with:
        node-version: 18.x

    - name: Define ToT chrome path
      if: matrix.chrome-channel == 'ToT'
      run: echo "CHROME_PATH=/home/runner/chrome-linux-tot/chrome" >> $GITHUB_ENV

    # Chrome Stable is already installed by default.
    - name: Install Chrome ToT
      if: matrix.chrome-channel == 'ToT'
      working-directory: /home/runner
      run: bash $GITHUB_WORKSPACE/core/scripts/download-chrome.sh && mv chrome-linux chrome-linux-tot

    - run: yarn install --frozen-lockfile --network-timeout 1000000
    - run: yarn build-report
    - run: yarn reset-link

    - run: sudo apt-get install xvfb
    - name: Run smoke tests
      run: |
        xvfb-run --auto-servernum yarn c8 yarn smoke --debug -j=2 --retries=2 --shard=${{ matrix.smoke-test-shard }}/$SHARD_TOTAL
        yarn c8 report --reporter text-lcov > smoke-coverage.lcov

    - name: Upload test coverage to Codecov
      if: matrix.chrome-channel == 'ToT'
      uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d
      with:
        flags: smoke
        file: ./smoke-coverage.lcov

    # Fail if any changes were written to source files.
    - run: git diff --exit-code

    - name: Upload failures
      if: failure()
      uses: actions/upload-artifact@v1
      with:
        name: Smokehouse (ubuntu; chrome ${{ matrix.chrome-channel }})
        path: .tmp/smokehouse-failures/

  smoke-windows:
    strategy:
      matrix:
        smoke-test-shard: [1, 2]
      # e.g. if set 1 fails, continue with set 2 anyway
      fail-fast: false
    runs-on: windows-latest
    name: Windows smoke ${{ matrix.smoke-test-shard }}/2

    steps:
    - name: git clone
      uses: actions/checkout@v3

    # Use Node 18 here earlier than everywhere else, see https://github.com/GoogleChrome/lighthouse/issues/15160#issuecomment-1589913408
    - name: Use Node.js 18.x
      uses: actions/setup-node@v3
      with:
        node-version: 18.x

    - name: Define ToT chrome path
      run: echo "CHROME_PATH=${env:GITHUB_WORKSPACE}\chrome-win\chrome.exe" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append

    # Chrome Stable is already installed by default.
    - name: Install Chrome ToT
      run: bash ${env:GITHUB_WORKSPACE}\core\scripts\download-chrome.sh

    - run: yarn install --frozen-lockfile --network-timeout 1000000
    - run: yarn build-report

    - name: Run smoke tests
      # Windows bots are slow, so only run enough tests to verify matching behavior.
      run: yarn smoke --debug -j=2 --retries=5 --shard=${{ matrix.smoke-test-shard }}/2 dbw oopif offline lantern metrics

    # Fail if any changes were written to source files.
    - run: git diff --exit-code

    - name: Upload failures
      if: failure()
      uses: actions/upload-artifact@v1
      with:
        name: Smokehouse (windows)
        path: .tmp/smokehouse-failures/

  smoke-bundle:
    strategy:
      matrix:
        smoke-test-shard: [1, 2, 3]
      # e.g. if set 1 fails, continue with set 2 anyway
      fail-fast: false
    runs-on: ubuntu-latest
    env:
      # The total number of shards. Set dynamically when length of *single* matrix variable is
      # computable. See https://git.luolix.topmunity/t/get-length-of-strategy-matrix-or-get-all-matrix-options/18342
      SHARD_TOTAL: 3
      FORCE_COLOR: true
    name: Bundled Lighthouse ${{ matrix.smoke-test-shard }}/3

    steps:
    - name: git clone
      uses: actions/checkout@v3

    - name: Use Node.js 18.x
      uses: actions/setup-node@v3
      with:
        node-version: 18.x

    - run: yarn install --frozen-lockfile --network-timeout 1000000
    - run: yarn build-report
    - run: yarn build-devtools

    - name: Define ToT chrome path
      run: echo "CHROME_PATH=/home/runner/chrome-linux-tot/chrome" >> $GITHUB_ENV

    - name: Install Chrome ToT
      working-directory: /home/runner
      run: bash $GITHUB_WORKSPACE/core/scripts/download-chrome.sh && mv chrome-linux chrome-linux-tot

    - run: sudo apt-get install xvfb
    - name: yarn test-bundle
      run: xvfb-run --auto-servernum yarn test-bundle --shard=${{ matrix.smoke-test-shard }}/$SHARD_TOTAL --retries=2

    # Fail if any changes were written to source files.
    - run: git diff --exit-code

    - name: Upload failures
      if: failure()
      uses: actions/upload-artifact@v1
      with:
        name: Smokehouse (bundled)
        path: .tmp/smokehouse-failures/