From 4121a69c27c6059d16ee4fca00537fa38630b3fa Mon Sep 17 00:00:00 2001 From: Mehul Kar Date: Tue, 21 May 2024 11:57:39 -0500 Subject: [PATCH] chore(ci): breakup GH test.yml (#8027) turbopack and turborepo tests are often blocking each other and there is no good reason to keep them all in one workflow. This PR separates them. Since linting and JS packages are not cleanly separated between the projects, they get their own workflows as well. --- .github/workflows/lint.yml | 136 ++++++ .github/workflows/test-js-packages.yml | 103 ++++ .../{test.yml => turbopack-test.yml} | 449 +----------------- .github/workflows/turborepo-test.yml | 393 +++++++++++++++ 4 files changed, 639 insertions(+), 442 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/test-js-packages.yml rename .github/workflows/{test.yml => turbopack-test.yml} (59%) create mode 100644 .github/workflows/turborepo-test.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000000000..ba84099593b26 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,136 @@ +name: Lint +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + determine_jobs: + name: Determine jobs to run + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: CI related changes + id: ci + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + .github/actions/** + .github/workflows/lint.yml + + - name: Rust related changes + id: rust + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + pnpm-lock.yaml + package.json + Cargo.** + crates/** + shim/** + xtask/** + .cargo/** + rust-toolchain + !**.md + !**.mdx + + - name: Formatting related changes + id: format + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + **/*.{yml,yaml,md,mdx,js,jsx,ts,tsx,json,toml,css} + + outputs: + rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} + format: ${{ steps.ci.outputs.diff != '' || steps.format.outputs.diff != '' }} + + rust_lint: + needs: [determine_jobs] + if: needs.determine_jobs.outputs.rust == 'true' + name: Rust lints + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Rust + uses: ./.github/actions/setup-rust + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo fmt check + run: | + cargo fmt --check + + - name: Check Cargo.toml formatting (taplo) + run: npx @taplo/cli@0.5.2 format --check + + - name: Check licenses + uses: EmbarkStudios/cargo-deny-action@v1 + with: + command: check licenses + + format_lint: + name: Formatting + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + needs: determine_jobs + if: needs.determine_jobs.outputs.format == 'true' + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: "Setup Node" + uses: ./.github/actions/setup-node + with: + extra-flags: --no-optional + node-version: "20" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Lint + # Filters some packages out, but not sure why + run: | + turbo run lint \ + --filter=!@vercel/devlow-bench \ + --filter=!@vercel/experimental-nft-next-plugin \ + --filter=!@vercel/experimental-nft-next-plugin \ + --filter=!turbopack-bump-action \ + --filter=!next-integration-stat \ + --env-mode=strict + + cleanup: + name: Cleanup + needs: + - rust_lint + - format_lint + if: always() + uses: ./.github/workflows/pr-clean-caches.yml + secrets: inherit diff --git a/.github/workflows/test-js-packages.yml b/.github/workflows/test-js-packages.yml new file mode 100644 index 0000000000000..0f0b68dd31251 --- /dev/null +++ b/.github/workflows/test-js-packages.yml @@ -0,0 +1,103 @@ +name: JS Package Tests +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + determine_jobs: + name: Determine jobs to run + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: CI related changes + id: ci + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + .github/actions/** + .github/workflows/test-js-packages.yml + + - name: /packages related changes + id: packages + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + packages/** + + - name: Docs related changes + id: docs + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + docs/** + + outputs: + ci: ${{ steps.ci.outputs.diff != ''}} + packages: ${{ steps.packages.outputs.diff != '' }} + docs: ${{ steps.docs.outputs.diff != '' }} + + js_packages: + name: JS Package Tests + timeout-minutes: 30 + if: needs.determine_jobs.outputs.ci == 'true' || needs.determine_jobs.outputs.packages == 'true' || needs.determine_jobs.outputs.docs == 'true' + needs: [determine_jobs] + runs-on: ${{ matrix.os.runner }} + strategy: + fail-fast: false + matrix: + os: + - name: ubuntu + runner: + - "self-hosted" + - "linux" + - "x64" + - "metal" + - name: macos + runner: macos-12 + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ vars.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + + steps: + # on main -> current + prev commit + # pr -> pr commits + base commit + - name: Determine fetch depth + id: fetch-depth + run: | + echo "depth=$(( ${{ github.event.pull_request.commits || 1 }} + 1 ))" >> $GITHUB_OUTPUT + + - name: Checkout + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + fetch-depth: ${{ steps.fetch-depth.outputs.depth }} + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + node-version: "20" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Run tests + # We manually set TURBO_API to an empty string to override Hetzner env + # We filter out turborepo-repository because it's a native package and needs + # to run when turbo core changes. This job (`js_packages`) does not run on turborpeo core + # changes, and we don't want to enable that beahvior for _all_ our JS packages. + run: | + TURBO_API= turbo run check-types test --filter=docs --filter="!turborepo-repository" --filter={./packages/*}...[${{ github.event.pull_request.base.sha || 'HEAD^1' }}] --color --env-mode=strict diff --git a/.github/workflows/test.yml b/.github/workflows/turbopack-test.yml similarity index 59% rename from .github/workflows/test.yml rename to .github/workflows/turbopack-test.yml index 2274e9a53aa1d..b5313e8d62acc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/turbopack-test.yml @@ -1,4 +1,4 @@ -name: Test +name: Turbopack Test on: push: branches: [main] @@ -55,7 +55,9 @@ jobs: with: PATTERNS: | .github/actions/** - .github/workflows/test.yml + .github/workflows/turbopack-test.yml + .github/workflows/test-turbopack-rust-bench-test.yml + .github/workflows/pr-clean-caches.yml - name: Root cargo related changes id: cargo @@ -65,13 +67,6 @@ jobs: Cargo.* rust-toolchain - - name: Turborepo version changes - id: turborepo_version - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - version.txt - - name: Rust related changes id: rust uses: technote-space/get-diff-action@v6 @@ -131,39 +126,6 @@ jobs: !**.md !**.mdx - - name: Turborepo integration tests changes - id: turborepo_integration - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - turborepo-tests/integration/** - turborepo-tests/helpers/** - - - name: Examples related changes - id: examples - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - examples/** - turborepo-tests/example-*/** - turborepo-tests/helpers/** - !**.md - !**.mdx - - - name: Turborepo JS Package related changes - id: turborepo_js - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - packages/** - - - name: Docs related changes - id: docs - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - docs/** - - name: Formatting related changes id: format uses: technote-space/get-diff-action@v6 @@ -172,227 +134,13 @@ jobs: **/*.{yml,yaml,md,mdx,js,jsx,ts,tsx,json,toml,css} outputs: + turbopack_typescript: ${{ steps.ci.outputs.diff != '' || steps.turbopack_typescript.outputs.diff != '' }} rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} - cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turbopack.outputs.diff == '' && steps.turborepo_rust.outputs.diff == '') }} + turbopack: ${{ steps.ci.outputs.diff != '' || steps.turbopack.outputs.diff != '' }} # We only test workspace dependency changes on main, not on PRs to speed up CI cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }} - turbopack: ${{ steps.ci.outputs.diff != '' || steps.turbopack.outputs.diff != '' }} - turbopack_typescript: ${{ steps.ci.outputs.diff != '' || steps.turbopack_typescript.outputs.diff != '' }} - turborepo_rust: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' }} + cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turbopack.outputs.diff == '' && steps.turborepo_rust.outputs.diff == '') }} turbopack_bench: ${{ steps.ci.outputs.diff != '' || steps.turbopack_bench.outputs.diff != '' }} - turborepo_build: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != ''}} - turborepo_integration: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != '' }} - examples: ${{ steps.ci.outputs.diff != '' || steps.examples.outputs.diff != '' || steps.turborepo_version.outputs.diff != '' }} - turborepo_js: ${{ steps.ci.outputs.diff != '' || steps.turborepo_js.outputs.diff != '' }} - docs: ${{ steps.ci.outputs.diff != '' || steps.docs.outputs.diff != '' }} - format: ${{ steps.ci.outputs.diff != '' || steps.format.outputs.diff != '' }} - push: ${{ steps.ci.outputs.diff != '' || github.event_name == 'push' }} - tag: ${{ steps.ci.outputs.diff != '' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/node-file-trace')) }} - main_push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - tag_push: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/node-file-trace') }} - - build_turborepo: - name: Build Turborepo - needs: determine_jobs - if: needs.determine_jobs.outputs.turborepo_build == 'true' - runs-on: ${{ matrix.os.runner }} - strategy: - fail-fast: false - matrix: - os: - - name: ubuntu - runner: - - "self-hosted" - - "linux" - - "x64" - - "metal" - - name: macos - runner: macos-latest - - name: windows - runner: windows-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Run sccache-cache - uses: mozilla-actions/sccache-action@v0.0.3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - windows: ${{ matrix.os.runner == 'windows-latest' }} - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - # We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues - - run: | - if [ -z "${RUSTC_WRAPPER}" ]; then - unset RUSTC_WRAPPER - fi - turbo run build --filter=cli --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} - shell: bash - env: - SCCACHE_BUCKET: turborepo-sccache - SCCACHE_REGION: us-east-2 - # Only use sccache if we're in the Vercel repo. - RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }} - CARGO_INCREMENTAL: 0 - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Run sccache stat for check - shell: bash - run: ${SCCACHE_PATH} --show-stats - if: ${{ !github.event.pull_request.head.repo.fork }} - - turborepo_integration: - name: Turborepo Integration - needs: [determine_jobs, build_turborepo] - if: needs.determine_jobs.outputs.turborepo_integration == 'true' - runs-on: ${{ matrix.os.runner }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - os: - - runner: ubuntu-latest - - runner: macos-latest - - runner: windows-latest - steps: - # On Windows, set autocrlf to input so that when the repo is cloned down - # the fixtures retain their line endings and don't get updated to CRLF. - # We want this because this repo also contains the fixtures for our test cases - # and these fixtures have files that need stable file hashes. If we let git update - # the line endings on checkout, the file hashes will change. - # https://www.git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf - - name: set crlf - if: matrix.os.runner == 'windows-latest' - shell: bash - run: git config --global core.autocrlf input - - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - windows: ${{ matrix.os.runner == 'windows-latest' }} - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Setup Graphviz - uses: ts-graphviz/setup-graphviz@v1 - with: - macos-skip-brew-update: "true" - env: - HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: true - - - name: Cache Prysk - id: cache-prysk - uses: actions/cache@v3 - with: - path: cli/.cram_env - key: prysk-venv-${{ matrix.os.runner }} - - - name: Integration Tests - run: turbo run test --filter=turborepo-tests-integration --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} - - turborepo_examples: - name: Turborepo Examples - needs: [determine_jobs] - if: needs.determine_jobs.outputs.examples == 'true' - timeout-minutes: 40 - - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - # Disable corepack. actions/setup-node invokes other package managers and - # that causes corepack to throw an error, so we disable it first. - - name: Disable corepack - shell: bash - run: corepack disable - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Check examples - shell: bash - # Concurrency being 1 is a big hammer here. - # It's a quick fix for non-deterministic behaviors we're seeing around package resolution. - # We could likely do some hacking to reparallelize this, but it's not worth it right now. - run: turbo run test --filter="@turborepo-examples-tests/*" --continue --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} --env-mode=strict --concurrency=1 - - # Disable corepack again. actions/setup-node's "Post" step runs at the end of - # this job invokes other package managers, and corepack throws an error. - # (corepack was enabled from inside the tests above). - - name: Disable corepack again - shell: bash - run: corepack disable - - js_packages: - name: JS Package Tests - timeout-minutes: 30 - if: needs.determine_jobs.outputs.turborepo_js == 'true' || needs.determine_jobs.outputs.docs == 'true' - needs: [determine_jobs] - runs-on: ${{ matrix.os.runner }} - strategy: - fail-fast: false - matrix: - os: - - name: ubuntu - runner: - - "self-hosted" - - "linux" - - "x64" - - "metal" - - name: macos - runner: macos-latest - env: - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - - steps: - # on main -> current + prev commit - # pr -> pr commits + base commit - - name: Determine fetch depth - id: fetch-depth - run: | - echo "depth=$(( ${{ github.event.pull_request.commits || 1 }} + 1 ))" >> $GITHUB_OUTPUT - - - name: Checkout - uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - fetch-depth: ${{ steps.fetch-depth.outputs.depth }} - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - node-version: "20" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Run tests - # We manually set TURBO_API to an empty string to override Hetzner env - # We filter out turborepo-repository because it's a native package and needs - # to run when turbo core changes. This job (`js_packages`) does not run on turborpeo core - # changes, and we don't want to enable that beahvior for _all_ our JS packages. - run: | - TURBO_API= turbo run check-types test --filter=docs --filter="!turborepo-repository" --filter={./packages/*}...[${{ github.event.pull_request.base.sha || 'HEAD^1' }}] --color --env-mode=strict turbopack_typescript: name: Turbopack TypeScript files @@ -425,87 +173,6 @@ jobs: working-directory: crates/turbopack-ecmascript-runtime/js run: pnpm run check - rust_lint: - needs: [determine_jobs] - if: needs.determine_jobs.outputs.rust == 'true' - name: Rust lints - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Rust - uses: ./.github/actions/setup-rust - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run cargo fmt check - run: | - cargo fmt --check - - - name: Check Cargo.toml formatting (taplo) - run: npx @taplo/cli@0.5.2 format --check - - - name: Check licenses - uses: EmbarkStudios/cargo-deny-action@v1 - with: - command: check licenses - - turborepo_rust_check: - needs: [determine_jobs] - # We test dependency changes only on main - if: | - (needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.turborepo_rust == 'true') || - needs.determine_jobs.outputs.cargo_on_main == 'true' || - needs.determine_jobs.outputs.cargo_only == 'true' - name: Turborepo rust check - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run cargo check - run: | - cargo groups check turborepo-libraries --features rustls-tls - - turborepo_rust_clippy: - needs: [turborepo_rust_check] - name: Turborepo rust clippy - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run cargo clippy - run: | - cargo groups clippy turborepo-libraries --features rustls-tls -- --deny clippy::all - - - name: Run ast-grep lints - run: | - npx --package @ast-grep/cli -- ast-grep scan $(cargo groups list turborepo-libraries | awk '{ print $2 }' | tr '\n' ' ') - turbopack_rust_check: needs: [determine_jobs] # We test dependency changes only on main @@ -639,48 +306,6 @@ jobs: filePath: ./out.log comment_tag: check_next_swc_turbopack - turborepo_rust_test: - needs: [turborepo_rust_check] - strategy: - fail-fast: false - matrix: - os: - - name: ubuntu - runner: - - "self-hosted" - - "linux" - - "x64" - - "metal" - nextest: linux - - name: macos - runner: macos-latest - nextest: mac - - name: windows - runner: windows-latest - nextest: windows-tar - runs-on: ${{ matrix.os.runner }} - name: Turborepo Rust testing on ${{ matrix.os.name }} - steps: - - name: Set git to use LF line endings - run: | - git config --global core.autocrlf false - git config --global core.eol lf - if: matrix.os.name == 'windows' - - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - windows: ${{ matrix.os.runner == 'windows-latest' }} - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Run tests - timeout-minutes: 120 - run: | - cargo groups test turborepo-libraries - turbopack_rust_test1: needs: [turbopack_rust_check] runs-on: @@ -844,53 +469,15 @@ jobs: os: ${{ matrix.os.name }} all: ${{ needs.determine_jobs.outputs.turbopack_bench == 'true' }} - format_lint: - name: Formatting - runs-on: - - "self-hosted" - - "linux" - - "x64" - - "metal" - needs: determine_jobs - if: needs.determine_jobs.outputs.format == 'true' - env: - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ vars.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Setup Turborepo Environment - uses: ./.github/actions/setup-turborepo-environment - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Install Global Turbo - uses: ./.github/actions/install-global-turbo - - - name: Lint - # Filters some workspaces out: - # - Other `@vercel/*` packages because ?? - run: turbo run lint --filter=!@vercel/devlow-bench --filter=!@vercel/experimental-nft-next-plugin --filter=!@vercel/experimental-nft-next-plugin --filter=!turbopack-bump-action --filter=!next-integration-stat --env-mode=strict - final: name: Ok needs: - determine_jobs - - turborepo_examples - - turborepo_integration - - js_packages - - rust_lint - - turborepo_rust_check - - turborepo_rust_clippy - turbopack_rust_check - turbopack_rust_clippy - turbopack_rust_test1 - turbopack_rust_test_bench1 - turbopack_typescript - - turborepo_rust_test - - format_lint if: always() permissions: contents: read @@ -914,19 +501,11 @@ jobs: fi } subjob ${{needs.determine_jobs.result}} "Determining jobs" - subjob ${{needs.turborepo_examples.result}} "Turborepo examples" - subjob ${{needs.turborepo_integration.result}} "Turborepo integration tests" - subjob ${{needs.js_packages.result}} "JS Package tests" - subjob ${{needs.rust_lint.result}} "Rust lints" - subjob ${{needs.turborepo_rust_check.result}} "Turborepo Rust checks" - subjob ${{needs.turborepo_rust_clippy.result}} "Turborepo Rust clippy" subjob ${{needs.turbopack_rust_check.result}} "Turbopack Rust checks" subjob ${{needs.turbopack_rust_clippy.result}} "Turbopack Rust clippy" subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)" subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)" subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks" - subjob ${{needs.turborepo_rust_test.result}} "Turborepo Rust tests" - subjob ${{needs.format_lint.result}} "Formatting" if [ "$cancelled" = "true" ]; then echo "cancelled=true" >> $GITHUB_OUTPUT elif [ "$failure" = "true" ]; then @@ -1026,11 +605,6 @@ jobs: needs: - final - determine_jobs - - turborepo_examples - - turborepo_integration - - rust_lint - - turborepo_rust_check - - turborepo_rust_clippy - turbopack_rust_check - turbopack_rust_clippy - turbopack_rust_test1 @@ -1038,8 +612,6 @@ jobs: - turbopack_rust_test_bench1 - turbopack_rust_test_bench2 - turbopack_typescript - - turborepo_rust_test - - format_lint if: always() permissions: contents: read @@ -1064,11 +636,6 @@ jobs: fi } subjob ${{needs.determine_jobs.result}} "Determining jobs" - subjob ${{needs.turborepo_examples.result}} "Turborepo examples" - subjob ${{needs.turborepo_integration.result}} "Turborepo integration tests" - subjob ${{needs.rust_lint.result}} "Rust lints" - subjob ${{needs.turborepo_rust_check.result}} "Turborepo Rust checks" - subjob ${{needs.turborepo_rust_clippy.result}} "Turborepo Rust clippy" subjob ${{needs.turbopack_rust_check.result}} "Turbopack Rust checks" subjob ${{needs.turbopack_rust_clippy.result}} "Turbopack Rust clippy" subjob ${{needs.turbopack_rust_test1.result}} "Turbopack Rust tests (linux)" @@ -1076,8 +643,6 @@ jobs: subjob ${{needs.turbopack_rust_test_bench1.result}} "Turbopack Rust benchmark tests (linux)" subjob ${{needs.turbopack_rust_test_bench2.result}} "Turbopack Rust benchmark tests (mac/win, non-blocking)" subjob ${{needs.turbopack_typescript.result}} "Turbopack Typescript checks" - subjob ${{needs.turborepo_rust_test.result}} "TurboRepo Rust tests" - subjob ${{needs.format_lint.result}} "Formatting" if [ "$cancelled" = "true" ]; then echo "cancelled=true" >> $GITHUB_OUTPUT elif [ "$failure" = "true" ]; then diff --git a/.github/workflows/turborepo-test.yml b/.github/workflows/turborepo-test.yml new file mode 100644 index 0000000000000..9317e74322e57 --- /dev/null +++ b/.github/workflows/turborepo-test.yml @@ -0,0 +1,393 @@ +name: Turborepo Test +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + actions: write + contents: read + pull-requests: read + +jobs: + determine_jobs: + name: Determine jobs to run + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: CI related changes + id: ci + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + .github/actions/** + .github/workflows/turborepo-test.yml + + - name: Root cargo related changes + id: cargo + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + Cargo.* + rust-toolchain + + - name: Turborepo version changes + id: turborepo_version + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + version.txt + + - name: Rust related changes + id: rust + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + pnpm-lock.yaml + package.json + Cargo.** + crates/** + shim/** + xtask/** + .cargo/** + rust-toolchain + !**.md + !**.mdx + + - name: Turborepo Rust related changes + id: turborepo_rust + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + pnpm-lock.yaml + package.json + crates/turborepo*/** + .cargo/** + rust-toolchain + !**.md + !**.mdx + + - name: Turborepo integration tests changes + id: turborepo_integration + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + turborepo-tests/integration/** + turborepo-tests/helpers/** + + - name: Examples related changes + id: examples + uses: technote-space/get-diff-action@v6 + with: + PATTERNS: | + examples/** + turborepo-tests/example-*/** + turborepo-tests/helpers/** + !**.md + !**.mdx + + outputs: + rust: ${{ steps.ci.outputs.diff != '' || steps.rust.outputs.diff != '' }} + cargo_only: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && steps.turborepo_rust.outputs.diff == '') }} + # We only test workspace dependency changes on main, not on PRs to speed up CI + cargo_on_main: ${{ steps.ci.outputs.diff != '' || (steps.cargo.outputs.diff != '' && github.event_name == 'push' && github.ref == 'refs/heads/main') }} + turborepo_rust: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' }} + turborepo_build: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != ''}} + turborepo_integration: ${{ steps.ci.outputs.diff != '' || steps.turborepo_rust.outputs.diff != '' || steps.turborepo_integration.outputs.diff != '' }} + examples: ${{ steps.ci.outputs.diff != '' || steps.examples.outputs.diff != '' || steps.turborepo_version.outputs.diff != '' }} + + build_turborepo: + name: Build Turborepo + needs: determine_jobs + if: needs.determine_jobs.outputs.turborepo_build == 'true' + runs-on: ${{ matrix.os.runner }} + strategy: + fail-fast: false + matrix: + os: + - name: ubuntu + runner: + - "self-hosted" + - "linux" + - "x64" + - "metal" + - name: macos + runner: macos-12 + - name: windows + runner: windows-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + windows: ${{ matrix.os.runner == 'windows-latest' }} + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + # We explicitly unset RUSTC_WRAPPER if it is an empty string as causes build issues + - run: | + if [ -z "${RUSTC_WRAPPER}" ]; then + unset RUSTC_WRAPPER + fi + turbo run build --filter=cli --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} + shell: bash + env: + SCCACHE_BUCKET: turborepo-sccache + SCCACHE_REGION: us-east-2 + # Only use sccache if we're in the Vercel repo. + RUSTC_WRAPPER: ${{ !github.event.pull_request.head.repo.fork && 'sccache' || '' }} + CARGO_INCREMENTAL: 0 + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Run sccache stat for check + shell: bash + run: ${SCCACHE_PATH} --show-stats + if: ${{ !github.event.pull_request.head.repo.fork }} + + integration: + name: Turborepo Integration + needs: [determine_jobs, build_turborepo] + if: needs.determine_jobs.outputs.turborepo_integration == 'true' + runs-on: ${{ matrix.os.runner }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - runner: ubuntu-latest + - runner: macos-12 + - runner: windows-latest + steps: + # On Windows, set autocrlf to input so that when the repo is cloned down + # the fixtures retain their line endings and don't get updated to CRLF. + # We want this because this repo also contains the fixtures for our test cases + # and these fixtures have files that need stable file hashes. If we let git update + # the line endings on checkout, the file hashes will change. + # https://www.git-scm.com/book/en/v2/Customizing-Git-Git-Configuration#_core_autocrlf + - name: set crlf + if: matrix.os.runner == 'windows-latest' + shell: bash + run: git config --global core.autocrlf input + - uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + windows: ${{ matrix.os.runner == 'windows-latest' }} + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Setup Graphviz + uses: ts-graphviz/setup-graphviz@v1 + with: + macos-skip-brew-update: "true" + env: + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: true + + - name: Cache Prysk + id: cache-prysk + uses: actions/cache@v3 + with: + path: cli/.cram_env + key: prysk-venv-${{ matrix.os.runner }} + + - name: Integration Tests + run: turbo run test --filter=turborepo-tests-integration --color --env-mode=strict --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} + + examples: + name: Turborepo Examples + needs: [determine_jobs] + if: needs.determine_jobs.outputs.examples == 'true' + timeout-minutes: 40 + + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + # Disable corepack. actions/setup-node invokes other package managers and + # that causes corepack to throw an error, so we disable it first. + - name: Disable corepack + shell: bash + run: corepack disable + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Install Global Turbo + uses: ./.github/actions/install-global-turbo + + - name: Check examples + shell: bash + # Concurrency being 1 is a big hammer here. + # It's a quick fix for non-deterministic behaviors we're seeing around package resolution. + # We could likely do some hacking to reparallelize this, but it's not worth it right now. + run: turbo run test --filter="@turborepo-examples-tests/*" --continue --token=${{ secrets.TURBO_TOKEN }} --team=${{ vars.TURBO_TEAM }} --env-mode=strict --concurrency=1 + + # Disable corepack again. actions/setup-node's "Post" step runs at the end of + # this job invokes other package managers, and corepack throws an error. + # (corepack was enabled from inside the tests above). + - name: Disable corepack again + shell: bash + run: corepack disable + + rust_lint: + needs: [determine_jobs] + if: needs.determine_jobs.outputs.rust == 'true' + name: Rust lints + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Rust + uses: ./.github/actions/setup-rust + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo fmt check + run: | + cargo fmt --check + + - name: Check Cargo.toml formatting (taplo) + run: npx @taplo/cli@0.5.2 format --check + + - name: Check licenses + uses: EmbarkStudios/cargo-deny-action@v1 + with: + command: check licenses + + rust_check: + needs: [determine_jobs] + # We test dependency changes only on main + if: | + (needs.determine_jobs.outputs.rust == 'true' && needs.determine_jobs.outputs.turborepo_rust == 'true') || + needs.determine_jobs.outputs.cargo_on_main == 'true' || + needs.determine_jobs.outputs.cargo_only == 'true' + name: Turborepo rust check + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo check + run: | + cargo groups check turborepo-libraries --features rustls-tls + + rust_clippy: + needs: [rust_check] + name: Turborepo rust clippy + runs-on: + - "self-hosted" + - "linux" + - "x64" + - "metal" + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run cargo clippy + run: | + cargo groups clippy turborepo-libraries --features rustls-tls -- --deny clippy::all + + - name: Run ast-grep lints + run: | + npx --package @ast-grep/cli -- ast-grep scan $(cargo groups list turborepo-libraries | awk '{ print $2 }' | tr '\n' ' ') + + rust_test: + needs: [rust_check] + strategy: + fail-fast: false + matrix: + os: + - name: ubuntu + runner: + - "self-hosted" + - "linux" + - "x64" + - "metal" + nextest: linux + - name: macos + runner: macos-12 + nextest: mac + - name: windows + runner: windows-latest + nextest: windows-tar + runs-on: ${{ matrix.os.runner }} + name: Turborepo Rust testing on ${{ matrix.os.name }} + steps: + - name: Set git to use LF line endings + run: | + git config --global core.autocrlf false + git config --global core.eol lf + if: matrix.os.name == 'windows' + + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Turborepo Environment + uses: ./.github/actions/setup-turborepo-environment + with: + windows: ${{ matrix.os.runner == 'windows-latest' }} + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Run tests + timeout-minutes: 120 + run: | + cargo groups test turborepo-libraries + + cleanup: + name: Cleanup + needs: + - determine_jobs + - build_turborepo + - integration + - examples + - rust_lint + - rust_check + - rust_clippy + - rust_test + if: always() + uses: ./.github/workflows/pr-clean-caches.yml + secrets: inherit