From b6c36aa224c99641c8b0dbe9e49cd4124e8c0f33 Mon Sep 17 00:00:00 2001 From: Tom French <15848336+TomAFrench@users.noreply.github.com> Date: Wed, 16 Aug 2023 22:18:51 +0100 Subject: [PATCH] chore(ci): Avoid writing to cache in workflows triggered by the merge queue (#2341) --- .github/workflows/publish.yml | 69 ++++++++++++++++++++++++----------- .github/workflows/test.yml | 25 +++++++++---- .github/workflows/wasm.yml | 56 ++++++++++++++++++++++++---- flake.nix | 1 + 4 files changed, 115 insertions(+), 36 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e047437eb7e..e8b80dfbb9d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -62,6 +62,12 @@ jobs: runs-on: macos-latest env: CROSS_CONFIG: ${{ github.workspace }}/.github/Cross.toml + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ strategy: matrix: target: [x86_64-apple-darwin, aarch64-apple-darwin] @@ -79,14 +85,10 @@ jobs: echo "SDKROOT=$(xcrun -sdk macosx$(sw_vers -productVersion) --show-sdk-path)" >> $GITHUB_ENV echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx$(sw_vers -productVersion) --show-sdk-platform-version)" >> $GITHUB_ENV - - uses: actions/cache@v3 + - uses: actions/cache/restore@v3 + id: cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -106,6 +108,13 @@ jobs: run: | cargo build --package nargo_cli --release --target ${{ matrix.target }} --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + # Don't create cache entries for the merge queue. + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + with: + path: ${{ env.CACHED_PATHS }} + key: ${{ steps.cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist @@ -144,6 +153,12 @@ jobs: runs-on: ubuntu-22.04 env: CROSS_CONFIG: ${{ github.workspace }}/.github/Cross.toml + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ strategy: fail-fast: false matrix: @@ -161,14 +176,10 @@ jobs: with: ref: ${{ inputs.tag || env.GITHUB_REF }} - - uses: actions/cache@v3 + - uses: actions/cache/restore@v3 + id: cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -189,6 +200,13 @@ jobs: cargo install cross --force --git https://github.com/cross-rs/cross cross build --package nargo_cli --release --target=${{ matrix.target }} --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + # Don't create cache entries for the merge queue. + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + with: + path: ${{ env.CACHED_PATHS }} + key: ${{ steps.cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist @@ -227,6 +245,12 @@ jobs: runs-on: windows-2022 env: CROSS_CONFIG: ${{ github.workspace }}/.github/Cross.toml + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ strategy: matrix: target: [x86_64-pc-windows-msvc] @@ -237,14 +261,10 @@ jobs: with: ref: ${{ inputs.tag || env.GITHUB_REF }} - - uses: actions/cache@v3 + - uses: actions/cache/restore@v3 + id: cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -264,6 +284,13 @@ jobs: run: | cargo build --package nargo_cli --release --target ${{ matrix.target }} --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + # Don't create cache entries for the merge queue. + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + with: + path: ${{ env.CACHED_PATHS }} + key: ${{ steps.cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 300f47fcd08..23a6716ab59 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,8 @@ jobs: name: Test on ${{ matrix.os }} runs-on: ${{ matrix.runner }} timeout-minutes: 30 + env: + CACHED_PATH: /tmp/nix-cache strategy: fail-fast: false @@ -37,20 +39,20 @@ jobs: name: barretenberg - name: Restore nix store cache - id: nix-store-cache - uses: actions/cache@v3 + uses: actions/cache/restore@v3 + id: cache with: - path: /tmp/nix-cache + path: ${{ env.CACHED_PATH }} key: ${{ runner.os }}-flake-${{ hashFiles('*.lock') }} # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 - name: Copy cache into nix store - if: steps.nix-store-cache.outputs.cache-hit == 'true' + if: steps.cache.outputs.cache-hit == 'true' # We don't check the signature because we're the one that created the cache run: | - for narinfo in /tmp/nix-cache/*.narinfo; do + for narinfo in ${{ env.CACHED_PATH }}/*.narinfo; do path=$(head -n 1 "$narinfo" | awk '{print $2}') - nix copy --no-check-sigs --from "file:///tmp/nix-cache" "$path" + nix copy --no-check-sigs --from "file://${{ env.CACHED_PATH }}" "$path" done - name: Run `nix flake check` @@ -58,6 +60,13 @@ jobs: nix flake check -L - name: Export cache from nix store - if: steps.nix-store-cache.outputs.cache-hit != 'true' + if: ${{ always() && steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} run: | - nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#native-cargo-artifacts + nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#native-cargo-artifacts + + - uses: actions/cache/save@v3 + # Write a cache entry even if the tests fail but don't create any for the merge queue. + if: ${{ always() && steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + with: + path: ${{ env.CACHED_PATH }} + key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 7d429accdbf..f052e9eb071 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -49,19 +49,22 @@ jobs: build-nargo: needs: [build-barretenberg] runs-on: ubuntu-22.04 + env: + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ steps: - name: Checkout Noir repo uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: actions/cache/restore@v3 + id: cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: ${{ env.CACHED_PATHS }} key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -79,6 +82,13 @@ jobs: run: | cargo build --package nargo_cli --release --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + # Don't create cache entries for the merge queue. + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + with: + path: ${{ env.CACHED_PATHS }} + key: ${{ steps.cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist @@ -94,6 +104,9 @@ jobs: build-wasm: runs-on: ubuntu-latest + env: + CACHED_PATH: /tmp/nix-cache + steps: - name: Checkout sources uses: actions/checkout@v3 @@ -104,10 +117,39 @@ jobs: nix_path: nixpkgs=channel:nixos-22.11 github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Restore nix store cache + uses: actions/cache/restore@v3 + id: cache + with: + path: ${{ env.CACHED_PATH }} + key: ${{ runner.os }}-flake-wasm-${{ hashFiles('*.lock') }} + + # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 + - name: Copy cache into nix store + if: steps.cache.outputs.cache-hit == 'true' + # We don't check the signature because we're the one that created the cache + run: | + for narinfo in ${{ env.CACHED_PATH }}/*.narinfo; do + path=$(head -n 1 "$narinfo" | awk '{print $2}') + nix copy --no-check-sigs --from "file://${{ env.CACHED_PATH }}" "$path" + done + - name: Build wasm package run: | nix build -L .#wasm + - name: Export cache from nix store + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + run: | + nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#noir-wasm-cargo-artifacts + + - uses: actions/cache/save@v3 + # Don't create cache entries for the merge queue. + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + with: + path: ${{ env.CACHED_PATH }} + key: ${{ steps.cache.outputs.cache-primary-key }} + - name: Dereference symlink run: echo "UPLOAD_PATH=$(readlink -f result)" >> $GITHUB_ENV diff --git a/flake.nix b/flake.nix index d77f17dd057..5514dc767df 100644 --- a/flake.nix +++ b/flake.nix @@ -254,6 +254,7 @@ # We expose the `*-cargo-artifacts` derivations so we can cache our cargo dependencies in CI inherit native-cargo-artifacts; inherit wasm-cargo-artifacts; + inherit noir-wasm-cargo-artifacts; }; # TODO(#1197): Look into installable apps with Nix flakes