integration-workflow.yml

on:
  push:
    branches:
    - master
    - 'cherry-pick/**'
  pull_request:
  merge_group:

concurrency:
  # group by workflow and ref; the last slightly strange component ensures that for pull
  # requests, we limit to 1 concurrent job, but for the master branch we don't
  group: ${{github.workflow}}-${{github.ref}}-${{github.ref != 'refs/heads/master' || github.run_number}}
  # Cancel intermediate builds, but only if it is a pull request build.
  cancel-in-progress: ${{startsWith(github.ref, 'refs/pull/')}}

name: 'Integration'
jobs:
  chore:
    name: 'Testing chores'
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
    - run: |
        git fetch --no-tags --unshallow origin HEAD master

    - uses: actions/setup-node@v3
      with:
        node-version: 18.x

    - name: 'Check that the Yarn files don''t change on new installs (fix w/ "yarn install")'
      run: |
        node ./scripts/run-yarn.js --immutable --immutable-cache
      shell: bash

    - name: 'Check that the cache files are consistent with their remote sources'
      run: |
        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- '.yarn/cache' 'packages/yarnpkg-libzip' | wc -l) -gt 0 ]]; then
          node ./scripts/run-yarn.js --immutable --immutable-cache --check-cache
        fi
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''
      env:
        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

    - name: 'Check that the patch files are consistent with fresh builds'
      run: |
        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- packages/plugin-compat/sources/patches | wc -l) -gt 0 ]]; then
          for generator in packages/plugin-compat/extra/*/gen-*-patch.sh; do
            bash $generator;
          done
          for generator in packages/plugin-compat/extra/*/gen-*-patch.js; do
            yarn node $generator;
          done
          [[ $(git diff --name-only packages/plugin-compat/sources/patches "packages/plugin-compat/extra/typescript/patch*" | tee /dev/stderr | wc -l) -eq 0 ]]
        fi
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''
      env:
        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

    - name: 'Check that libzip artifacts are consistent with a fresh build (fix w/ "git merge master && bash packages/yarnpkg-libzip/artifacts/build.sh")'
      run: |
        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- 'packages/yarnpkg-libzip/artifacts' 'packages/yarnpkg-libzip/sources/*.js' | wc -l) -gt 0 ]]; then
          bash packages/yarnpkg-libzip/artifacts/build.sh
          [[ $(git diff --name-only 'packages/yarnpkg-libzip/artifacts' 'packages/yarnpkg-libzip/sources/*.js' | wc -l) -eq 0 ]]
        fi
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''
      env:
        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

    - name: 'Check that the PnP hooks are consistent with a fresh build (fix w/ "git merge master && yarn update:pnp:hook")'
      run: |
        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- 'packages/yarnpkg-{fslib,libzip,pnp}/sources/**/*' | wc -l) -gt 0 ]]; then
          node ./scripts/run-yarn.js build:pnp:hook
          [[ $(git diff --name-only packages/yarnpkg-pnp/sources/{hook.js,esm-loader/built-loader.js} | wc -l) -eq 0 ]]
        fi
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''
      env:
        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

    - name: 'Check that the Zip worker is consistent with a fresh build (fix w/ "git merge master && yarn update:zip:worker")'
      run: |
        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- 'packages/yarnpkg-{fslib,libzip,core}/sources/**/*' | wc -l) -gt 0 ]]; then
          node ./scripts/run-yarn.js build:zip:worker
          [[ $(git diff --name-only packages/yarnpkg-core/sources/worker-zip/index.js | wc -l) -eq 0 ]]
        fi
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''
      env:
        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

    - name: 'Check that the pluginCommands file is consistent with a fresh build (fix w/ "yarn build:plugin-commands")'
      run: |
        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- packages/yarnpkg-cli/sources/pluginCommands.ts 'packages/*/sources/commands/**/*' | wc -l) -gt 0 ]]; then
          node ./scripts/run-yarn.js build:plugin-commands
          [[ $(git diff --name-only packages/yarnpkg-cli/sources/pluginCommands.ts | tee /dev/stderr | wc -l) -eq 0 ]]
        fi
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''
      env:
        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

    - name: 'Check that the grammars are consistent with fresh builds (fix w/ "yarn grammar:all")'
      run: |
        if [[ $(git diff --name-only "$(git merge-base origin/"$TARGET_BRANCH" HEAD)" HEAD -- 'packages/yarnpkg-parsers/sources/grammars/*.{pegjs,js}' | wc -l) -gt 0 ]]; then
          node ./scripts/run-yarn.js grammar:all
          [[ $(git diff --name-only packages/yarnpkg-parsers/sources/grammars/*.js | tee /dev/stderr | wc -l) -eq 0 ]]
        fi
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''
      env:
        TARGET_BRANCH: ${{github.event.pull_request.base.ref}}

    - name: 'Check that the PR describes which packages should be bumped (fix w/ "yarn version check -i")'
      run: |
        node ./scripts/run-yarn.js version check
      shell: bash
      if: |
        (success() || failure()) && github.event.pull_request != ''

    - name: 'Check for linting errors (fix w/ "yarn test:lint --fix")'
      run: |
        node ./scripts/run-yarn.js test:lint
      shell: bash
      if: |
        success() || failure()

    - name: 'Check for unmet constraints (fix w/ "yarn constraints --fix")'
      run: |
        node ./scripts/run-yarn.js constraints
      shell: bash
      if: |
        success() || failure()

    - name: 'Check for type errors'
      run: |
        node ./scripts/run-yarn.js typecheck:all
      shell: bash
      if: |
        success() || failure()

    - name: 'Check for duplicate dependencies (fix w/ "yarn dedupe")'
      run: |
        node ./scripts/run-yarn.js dedupe --check
      shell: bash
      if: |
        success() || failure()

  build:
    name: 'Build artifacts'
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3

    - uses: actions/setup-node@v3
      with:
        node-version: 18.x

    - name: 'Build bundle & plugins'
      run: |
        node --version
        node ./scripts/run-yarn.js build:pnp:hook
        node ./scripts/run-yarn.js build:zip:worker
        node ./scripts/run-yarn.js build:cli
        mv ./packages/yarnpkg-cli/bundles/yarn.js ./packages/yarnpkg-cli/bundles/yarn-min.js
        node ./scripts/run-yarn.js build:cli --no-minify
      shell: bash

    - uses: actions/upload-artifact@v3
      with:
        name: yarn-artifacts
        path: |
          ./packages/*/bundles
          ./packages/yarnpkg-pnp/sources/hook.js
          ./packages/yarnpkg-core/sources/worker-zip/index.js

    - name: 'Build vscode-zipfs'
      run: |
        node ./scripts/run-yarn.js package:vscode-zipfs
      if: |
        success() || failure()

    - uses: actions/upload-artifact@v3
      with:
        name: vscode-zipfs
        path: ./packages/vscode-zipfs/vscode-zipfs-*.vsix

  acceptance:
    strategy:
      fail-fast: false
      matrix:
        # We run the ubuntu tests on multiple Node versions with 2 shards since they're the fastest.
        # https://github.com/nodejs/node/issues/48444
        node: [18, 19, 20.2]
        platform: [ubuntu-latest]
        shard: ['1/2', '2/2']
        # We run the rest of the tests on the minimum Node version we support with 3 shards.
        include:
          # Windows tests
          - {node: 18, platform: windows-latest, shard: 1/3}
          - {node: 18, platform: windows-latest, shard: 2/3}
          - {node: 18, platform: windows-latest, shard: 3/3}
          # macOS tests
          - {node: 18, platform: macos-latest, shard: 1/3}
          - {node: 18, platform: macos-latest, shard: 2/3}
          - {node: 18, platform: macos-latest, shard: 3/3}

    name: '${{matrix.platform}} w/ Node.js ${{matrix.node}}.x (${{matrix.shard}})'
    runs-on: ${{matrix.platform}}
    needs: build

    steps:
    - uses: actions/checkout@v3

    - name: 'Use Node.js ${{matrix.node}}.x'
      uses: actions/setup-node@v3
      with:
        node-version: ${{matrix.node}}.x

    - uses: actions/download-artifact@v3
      with:
        name: yarn-artifacts
        path: packages

    - name: 'Run the integration tests'
      run: |
        node ./scripts/run-yarn.js test:integration --maxWorkers=100% --shard=${{matrix.shard}}
      shell: bash

    - name: 'Run the unit tests'
      run: |
        node ./scripts/run-yarn.js test:unit --maxWorkers=100% --shard=${{matrix.shard}}
      shell: bash
      if: |
        success() || failure()

  big-endian:
    strategy:
      fail-fast: false
      matrix:
        shard: ['1/4', '2/4', '3/4', '4/4']

    name: 'Big-endian debian w/ Node.js latest (${{matrix.shard}})'
    runs-on: ubuntu-latest
    if: |
      github.ref == 'refs/heads/master'
    needs: build

    steps:
    - uses: actions/checkout@v3

    - uses: actions/download-artifact@v3
      with:
        name: yarn-artifacts
        path: packages

    # https://github.com/multiarch/qemu-user-static
    - name: 'Enable execution of multi-arch containers'
      run: |
        docker run --rm --privileged multiarch/qemu-user-static --reset --persistent yes
      shell: bash

    # Apparently @babel/register is insanely slow inside the big-endian
    # docker container, so we don't want to run yarn from sources at all.

    - name: 'Run the integration tests'
      run: |
        docker run --rm --volume "$PWD:/berry" --env GITHUB_ACTIONS --env YARN_IGNORE_PATH=1 --workdir /berry s390x/node /bin/bash -c 'node ./packages/yarnpkg-cli/bundles/yarn.js test:integration --maxWorkers=100% --shard=${{matrix.shard}}'
      shell: bash

    - name: 'Run the unit tests'
      run: |
        docker run --rm --volume "$PWD:/berry" --env GITHUB_ACTIONS --env YARN_IGNORE_PATH=1 --workdir /berry s390x/node /bin/bash -c 'node ./packages/yarnpkg-cli/bundles/yarn.js test:unit --maxWorkers=100% --shard=${{matrix.shard}}'
      shell: bash
      if: |
        success() || failure()