From 322f2501af824d74ce1065f328ab6d28c018110f Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 13:48:27 +0100 Subject: [PATCH 01/19] chore(ci): use custom caching action to prevent cache writes in merge queue --- .github/actions/cache/action.yml | 38 ++++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 9 +++++--- .github/workflows/test.yml | 3 ++- .github/workflows/wasm.yml | 3 ++- 4 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 .github/actions/cache/action.yml diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml new file mode 100644 index 0000000000..8fe1513c49 --- /dev/null +++ b/.github/actions/cache/action.yml @@ -0,0 +1,38 @@ +name: Cache +description: 'A wrapper around `actions/cache` which exposes an additional `read-only` input to conditionally write new cache entries' + +inputs: + path: + description: 'A list of files, directories, and wildcard patterns to cache and restore' + required: true + key: + description: 'An explicit key for restoring and saving the cache' + required: true + read-only: + description: 'Do not write cache artifacts in the case of a missed cache' + default: 'false' + required: false + +outputs: + cache-hit: + description: 'A boolean value to indicate an exact match was found for the primary key' + value: ${{ steps.cache.outputs.cache-hit || steps.restore.outputs.cache-hit }} + +runs: + using: composite + steps: + + - id: restore + uses: actions/cache/restore@v3 + if: ${{ inputs.read_only }} + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + + - id: cache + uses: actions/cache@v3 + if: ${{ !inputs.read_only }} + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e047437eb7..769e508290 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -79,7 +79,7 @@ jobs: echo "SDKROOT=$(xcrun -sdk macosx$(sw_vers -productVersion) --show-sdk-path)" >> $GITHUB_ENV echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx$(sw_vers -productVersion) --show-sdk-platform-version)" >> $GITHUB_ENV - - uses: actions/cache@v3 + - uses: ./.github/actions/cache with: path: | ~/.cargo/bin/ @@ -88,6 +88,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} + read-only: ${{ github.event_name == 'merge_group' }} - name: Download artifact uses: actions/download-artifact@v3 @@ -161,7 +162,7 @@ jobs: with: ref: ${{ inputs.tag || env.GITHUB_REF }} - - uses: actions/cache@v3 + - uses: ./.github/actions/cache with: path: | ~/.cargo/bin/ @@ -170,6 +171,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} + read-only: ${{ github.event_name == 'merge_group' }} - name: Download artifact uses: actions/download-artifact@v3 @@ -237,7 +239,7 @@ jobs: with: ref: ${{ inputs.tag || env.GITHUB_REF }} - - uses: actions/cache@v3 + - uses: ./.github/actions/cache with: path: | ~/.cargo/bin/ @@ -246,6 +248,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} + read-only: ${{ github.event_name == 'merge_group' }} - name: Download artifact uses: actions/download-artifact@v3 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 300f47fcd0..c5f4f66aed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,10 +38,11 @@ jobs: - name: Restore nix store cache id: nix-store-cache - uses: actions/cache@v3 + uses: ./.github/actions/cache with: path: /tmp/nix-cache key: ${{ runner.os }}-flake-${{ hashFiles('*.lock') }} + read-only: ${{ github.event_name == 'merge_group' }} # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 - name: Copy cache into nix store diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 7d429accdb..bab132f941 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -54,7 +54,7 @@ jobs: - name: Checkout Noir repo uses: actions/checkout@v3 - - uses: actions/cache@v3 + - uses: ./.github/actions/cache with: path: | ~/.cargo/bin/ @@ -63,6 +63,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + read-only: ${{ github.event_name == 'merge_group' }} - name: Download artifact uses: actions/download-artifact@v3 From 12fe66e9e9933f523155fcf59fdeb76e9b6ff1dc Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 17:24:12 +0100 Subject: [PATCH 02/19] chore: replace custom action with explicit calls to restore and save --- .github/actions/cache/action.yml | 38 -------------------------------- .github/workflows/publish.yml | 33 +++++++++++++++++++++------ .github/workflows/test.yml | 15 ++++++++----- .github/workflows/wasm.yml | 10 +++++++-- 4 files changed, 44 insertions(+), 52 deletions(-) delete mode 100644 .github/actions/cache/action.yml diff --git a/.github/actions/cache/action.yml b/.github/actions/cache/action.yml deleted file mode 100644 index 8fe1513c49..0000000000 --- a/.github/actions/cache/action.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Cache -description: 'A wrapper around `actions/cache` which exposes an additional `read-only` input to conditionally write new cache entries' - -inputs: - path: - description: 'A list of files, directories, and wildcard patterns to cache and restore' - required: true - key: - description: 'An explicit key for restoring and saving the cache' - required: true - read-only: - description: 'Do not write cache artifacts in the case of a missed cache' - default: 'false' - required: false - -outputs: - cache-hit: - description: 'A boolean value to indicate an exact match was found for the primary key' - value: ${{ steps.cache.outputs.cache-hit || steps.restore.outputs.cache-hit }} - -runs: - using: composite - steps: - - - id: restore - uses: actions/cache/restore@v3 - if: ${{ inputs.read_only }} - with: - path: ${{ inputs.path }} - key: ${{ inputs.key }} - - - id: cache - uses: actions/cache@v3 - if: ${{ !inputs.read_only }} - with: - path: ${{ inputs.path }} - key: ${{ inputs.key }} - diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 769e508290..0bd48912e2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -79,7 +79,8 @@ jobs: echo "SDKROOT=$(xcrun -sdk macosx$(sw_vers -productVersion) --show-sdk-path)" >> $GITHUB_ENV echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx$(sw_vers -productVersion) --show-sdk-platform-version)" >> $GITHUB_ENV - - uses: ./.github/actions/cache + - uses: actions/cache/restore@v3 + id: restore-cache with: path: | ~/.cargo/bin/ @@ -88,8 +89,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - read-only: ${{ github.event_name == 'merge_group' }} - + - name: Download artifact uses: actions/download-artifact@v3 with: @@ -107,6 +107,12 @@ jobs: run: | cargo build --package nargo_cli --release --target ${{ matrix.target }} --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + with: + path: ${{ steps.restore-cache.inputs.path }} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist @@ -162,7 +168,8 @@ jobs: with: ref: ${{ inputs.tag || env.GITHUB_REF }} - - uses: ./.github/actions/cache + - uses: actions/cache/restore@v3 + id: restore-cache with: path: | ~/.cargo/bin/ @@ -171,7 +178,6 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - read-only: ${{ github.event_name == 'merge_group' }} - name: Download artifact uses: actions/download-artifact@v3 @@ -191,6 +197,12 @@ jobs: cargo install cross --force --git https://github.com/cross-rs/cross cross build --package nargo_cli --release --target=${{ matrix.target }} --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + with: + path: ${{ steps.restore-cache.inputs.path }} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist @@ -239,7 +251,8 @@ jobs: with: ref: ${{ inputs.tag || env.GITHUB_REF }} - - uses: ./.github/actions/cache + - uses: actions/cache/restore@v3 + id: restore-cache with: path: | ~/.cargo/bin/ @@ -248,7 +261,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - read-only: ${{ github.event_name == 'merge_group' }} + - name: Download artifact uses: actions/download-artifact@v3 @@ -267,6 +280,12 @@ jobs: run: | cargo build --package nargo_cli --release --target ${{ matrix.target }} --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + with: + path: ${{ steps.restore-cache.inputs.path }} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5f4f66aed..2ee1bc8263 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,16 +37,15 @@ jobs: name: barretenberg - name: Restore nix store cache - id: nix-store-cache - uses: ./.github/actions/cache + uses: actions/cache/restore@v3 + id: restore-cache with: path: /tmp/nix-cache key: ${{ runner.os }}-flake-${{ hashFiles('*.lock') }} - read-only: ${{ github.event_name == 'merge_group' }} - + # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 - name: Copy cache into nix store - if: steps.nix-store-cache.outputs.cache-hit == 'true' + if: steps.restore-cache.outputs.cache-hit == 'true' # We don't check the signature because we're the one that created the cache run: | for narinfo in /tmp/nix-cache/*.narinfo; do @@ -62,3 +61,9 @@ jobs: if: steps.nix-store-cache.outputs.cache-hit != 'true' run: | nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#native-cargo-artifacts + + - uses: actions/cache/save@v3 + if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + with: + path: ${{ steps.restore-cache.inputs.path }} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index bab132f941..9802fcf55c 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -54,7 +54,8 @@ jobs: - name: Checkout Noir repo uses: actions/checkout@v3 - - uses: ./.github/actions/cache + - uses: actions/cache/restore@v3 + id: restore-cache with: path: | ~/.cargo/bin/ @@ -63,7 +64,6 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - read-only: ${{ github.event_name == 'merge_group' }} - name: Download artifact uses: actions/download-artifact@v3 @@ -80,6 +80,12 @@ jobs: run: | cargo build --package nargo_cli --release --no-default-features --features plonk_bn254_wasm + - uses: actions/cache/save@v3 + if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + with: + path: ${{ steps.restore-cache.inputs.path }} + key: ${{ steps.restore-cache.outputs.cache-primary-key }} + - name: Package artifacts run: | mkdir dist From fec0ea321bcc00a1a34e3d1449963e7862d38e13 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 17:26:06 +0100 Subject: [PATCH 03/19] chore: whitespace changes --- .github/workflows/publish.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0bd48912e2..ffef39aecf 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -89,7 +89,7 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - + - name: Download artifact uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ee1bc8263..eb297c876a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,7 +42,7 @@ jobs: with: path: /tmp/nix-cache key: ${{ runner.os }}-flake-${{ hashFiles('*.lock') }} - + # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 - name: Copy cache into nix store if: steps.restore-cache.outputs.cache-hit == 'true' From ecfd5a6744215a75105658a02d6d9aa010c757c9 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 17:26:47 +0100 Subject: [PATCH 04/19] chore: whitespace changes --- .github/workflows/publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ffef39aecf..62e145d676 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -261,7 +261,6 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Download artifact uses: actions/download-artifact@v3 From 2dcdc1f6f359e18eb02b338b962fd2c3f687ee8f Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 17:46:42 +0100 Subject: [PATCH 05/19] chore: save cache entry even if nix tests fail --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index eb297c876a..99dabd9d3f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,8 @@ jobs: nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#native-cargo-artifacts - uses: actions/cache/save@v3 - if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + # Write a cache entry even if the tests fail but don't create any for the merge queue. + if: ${{ always() && github.event_name != 'merge_group' }} with: path: ${{ steps.restore-cache.inputs.path }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} From 0f9416f030c19834f45424c27c21cf5f908c1ee8 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 18:30:12 +0100 Subject: [PATCH 06/19] fix: only write cache on cache misses --- .github/workflows/publish.yml | 9 ++++++--- .github/workflows/test.yml | 2 +- .github/workflows/wasm.yml | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 62e145d676..a0792df05f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -108,7 +108,8 @@ jobs: cargo build --package nargo_cli --release --target ${{ matrix.target }} --no-default-features --features plonk_bn254_wasm - uses: actions/cache/save@v3 - if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + # Don't create cache entries for the merge queue. + if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ steps.restore-cache.inputs.path }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} @@ -198,7 +199,8 @@ jobs: cross build --package nargo_cli --release --target=${{ matrix.target }} --no-default-features --features plonk_bn254_wasm - uses: actions/cache/save@v3 - if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + # Don't create cache entries for the merge queue. + if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ steps.restore-cache.inputs.path }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} @@ -280,7 +282,8 @@ jobs: cargo build --package nargo_cli --release --target ${{ matrix.target }} --no-default-features --features plonk_bn254_wasm - uses: actions/cache/save@v3 - if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + # Don't create cache entries for the merge queue. + if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ steps.restore-cache.inputs.path }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99dabd9d3f..d4a3b3bf2b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: - uses: actions/cache/save@v3 # Write a cache entry even if the tests fail but don't create any for the merge queue. - if: ${{ always() && github.event_name != 'merge_group' }} + if: ${{ always() && steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ steps.restore-cache.inputs.path }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 9802fcf55c..7ab9e4fc0e 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -81,7 +81,8 @@ jobs: cargo build --package nargo_cli --release --no-default-features --features plonk_bn254_wasm - uses: actions/cache/save@v3 - if: ${{ github.event_name != 'merge_group' }} # Don't create cache entries for the merge queue. + # Don't create cache entries for the merge queue. + if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ steps.restore-cache.inputs.path }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} From 33795b9f41ab1f606e6af563fe6cbcc851c1c8eb Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 18:35:46 +0100 Subject: [PATCH 07/19] fix: remove usage of `steps[*].inputs` --- .github/workflows/publish.yml | 45 +++++++++++++++++++---------------- .github/workflows/test.yml | 2 +- .github/workflows/wasm.yml | 14 ++++++----- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a0792df05f..ce4c79035a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -62,6 +62,12 @@ jobs: runs-on: macos-latest env: CROSS_CONFIG: ${{ github.workspace }}/.github/Cross.toml + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ strategy: matrix: target: [x86_64-apple-darwin, aarch64-apple-darwin] @@ -82,12 +88,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: $CACHED_PATHS key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -111,7 +112,7 @@ jobs: # Don't create cache entries for the merge queue. if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: ${{ steps.restore-cache.inputs.path }} + path: $CACHED_PATHS key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Package artifacts @@ -152,6 +153,12 @@ jobs: runs-on: ubuntu-22.04 env: CROSS_CONFIG: ${{ github.workspace }}/.github/Cross.toml + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ strategy: fail-fast: false matrix: @@ -172,12 +179,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: $CACHED_PATHS key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -202,7 +204,7 @@ jobs: # Don't create cache entries for the merge queue. if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: ${{ steps.restore-cache.inputs.path }} + path: $CACHED_PATHS key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Package artifacts @@ -243,6 +245,12 @@ jobs: runs-on: windows-2022 env: CROSS_CONFIG: ${{ github.workspace }}/.github/Cross.toml + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ strategy: matrix: target: [x86_64-pc-windows-msvc] @@ -256,12 +264,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: $CACHED_PATHS key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -285,7 +288,7 @@ jobs: # Don't create cache entries for the merge queue. if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: ${{ steps.restore-cache.inputs.path }} + path: $CACHED_PATHS key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Package artifacts diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d4a3b3bf2b..3d80e4901e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,5 +66,5 @@ jobs: # Write a cache entry even if the tests fail but don't create any for the merge queue. if: ${{ always() && steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: ${{ steps.restore-cache.inputs.path }} + path: /tmp/nix-cache key: ${{ steps.restore-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 7ab9e4fc0e..727a5b96d4 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -49,6 +49,13 @@ jobs: build-nargo: needs: [build-barretenberg] runs-on: ubuntu-22.04 + env: + CACHED_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ steps: - name: Checkout Noir repo @@ -57,12 +64,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + path: $CACHED_PATHS key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact From 5d8cf30b76869626977def4e8dd6117f59697cd3 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 18:53:22 +0100 Subject: [PATCH 08/19] chore: use `env.CACHED_PATHS` form of environment variable --- .github/workflows/publish.yml | 12 ++++++------ .github/workflows/wasm.yml | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ce4c79035a..b9b4a1b544 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -88,7 +88,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: $CACHED_PATHS + path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -112,7 +112,7 @@ jobs: # Don't create cache entries for the merge queue. if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: $CACHED_PATHS + path: ${{ env.CACHED_PATHS }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Package artifacts @@ -179,7 +179,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: $CACHED_PATHS + path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -204,7 +204,7 @@ jobs: # Don't create cache entries for the merge queue. if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: $CACHED_PATHS + path: ${{ env.CACHED_PATHS }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Package artifacts @@ -264,7 +264,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: $CACHED_PATHS + path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact @@ -288,7 +288,7 @@ jobs: # Don't create cache entries for the merge queue. if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: $CACHED_PATHS + path: ${{ env.CACHED_PATHS }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Package artifacts diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 727a5b96d4..9042d6f039 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -64,7 +64,7 @@ jobs: - uses: actions/cache/restore@v3 id: restore-cache with: - path: $CACHED_PATHS + path: ${{ env.CACHED_PATHS }} key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Download artifact From 126621543cc34c1390fbf7d5ddf4f6d89e25bf5b Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 18:54:30 +0100 Subject: [PATCH 09/19] chore: indentation change --- .github/workflows/wasm.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 9042d6f039..6a5934372c 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -51,11 +51,11 @@ jobs: runs-on: ubuntu-22.04 env: CACHED_PATHS: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ steps: - name: Checkout Noir repo From 9e904897b2ef646bfa88f8c16449b66e609581e3 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 18:55:06 +0100 Subject: [PATCH 10/19] chore: cleanup remaining usage of `.inputs` --- .github/workflows/wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 6a5934372c..4fc5e919ba 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -86,7 +86,7 @@ jobs: # Don't create cache entries for the merge queue. if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: ${{ steps.restore-cache.inputs.path }} + path: ${{ env.CACHED_PATHS }} key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Package artifacts From afe5dadb681a61673e5bd26529b7fab469b15105 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 18:55:56 +0100 Subject: [PATCH 11/19] chore: `restore-cache` -> `cache` --- .github/workflows/publish.yml | 18 +++++++++--------- .github/workflows/test.yml | 8 ++++---- .github/workflows/wasm.yml | 6 +++--- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b9b4a1b544..12f57909e7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -86,7 +86,7 @@ jobs: echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx$(sw_vers -productVersion) --show-sdk-platform-version)" >> $GITHUB_ENV - uses: actions/cache/restore@v3 - id: restore-cache + id: cache with: path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} @@ -110,10 +110,10 @@ jobs: - uses: actions/cache/save@v3 # Don't create cache entries for the merge queue. - if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ env.CACHED_PATHS }} - key: ${{ steps.restore-cache.outputs.cache-primary-key }} + key: ${{ steps.cache.outputs.cache-primary-key }} - name: Package artifacts run: | @@ -177,7 +177,7 @@ jobs: ref: ${{ inputs.tag || env.GITHUB_REF }} - uses: actions/cache/restore@v3 - id: restore-cache + id: cache with: path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} @@ -202,10 +202,10 @@ jobs: - uses: actions/cache/save@v3 # Don't create cache entries for the merge queue. - if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ env.CACHED_PATHS }} - key: ${{ steps.restore-cache.outputs.cache-primary-key }} + key: ${{ steps.cache.outputs.cache-primary-key }} - name: Package artifacts run: | @@ -262,7 +262,7 @@ jobs: ref: ${{ inputs.tag || env.GITHUB_REF }} - uses: actions/cache/restore@v3 - id: restore-cache + id: cache with: path: ${{ env.CACHED_PATHS }} key: ${{ matrix.target }}-cargo-${{ hashFiles('**/Cargo.lock') }} @@ -286,10 +286,10 @@ jobs: - uses: actions/cache/save@v3 # Don't create cache entries for the merge queue. - if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ env.CACHED_PATHS }} - key: ${{ steps.restore-cache.outputs.cache-primary-key }} + key: ${{ steps.cache.outputs.cache-primary-key }} - name: Package artifacts run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3d80e4901e..0e5e630cc0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,14 +38,14 @@ jobs: - name: Restore nix store cache uses: actions/cache/restore@v3 - id: restore-cache + id: cache with: path: /tmp/nix-cache key: ${{ runner.os }}-flake-${{ hashFiles('*.lock') }} # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 - name: Copy cache into nix store - if: steps.restore-cache.outputs.cache-hit == 'true' + if: steps.cache.outputs.cache-hit == 'true' # We don't check the signature because we're the one that created the cache run: | for narinfo in /tmp/nix-cache/*.narinfo; do @@ -64,7 +64,7 @@ jobs: - uses: actions/cache/save@v3 # Write a cache entry even if the tests fail but don't create any for the merge queue. - if: ${{ always() && steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + if: ${{ always() && steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: /tmp/nix-cache - key: ${{ steps.restore-cache.outputs.cache-primary-key }} + key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 4fc5e919ba..f0c432515a 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -62,7 +62,7 @@ jobs: uses: actions/checkout@v3 - uses: actions/cache/restore@v3 - id: restore-cache + id: cache with: path: ${{ env.CACHED_PATHS }} key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} @@ -84,10 +84,10 @@ jobs: - uses: actions/cache/save@v3 # Don't create cache entries for the merge queue. - if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ env.CACHED_PATHS }} - key: ${{ steps.restore-cache.outputs.cache-primary-key }} + key: ${{ steps.cache.outputs.cache-primary-key }} - name: Package artifacts run: | From fd345c92eee832aeb5b635885319584c36bb68bf Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 19:05:11 +0100 Subject: [PATCH 12/19] chore: use env var in `test.yaml` workflow --- .github/workflows/test.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0e5e630cc0..a7e7277d00 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,8 @@ jobs: name: Test on ${{ matrix.os }} runs-on: ${{ matrix.runner }} timeout-minutes: 30 + env: + CACHED_PATH: /tmp/nix-cache strategy: fail-fast: false @@ -40,7 +42,7 @@ jobs: uses: actions/cache/restore@v3 id: cache with: - path: /tmp/nix-cache + path: ${{ env.CACHED_PATH }} key: ${{ runner.os }}-flake-${{ hashFiles('*.lock') }} # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 @@ -48,9 +50,9 @@ jobs: if: steps.cache.outputs.cache-hit == 'true' # We don't check the signature because we're the one that created the cache run: | - for narinfo in /tmp/nix-cache/*.narinfo; do + for narinfo in ${{ env.CACHED_PATH }}/*.narinfo; do path=$(head -n 1 "$narinfo" | awk '{print $2}') - nix copy --no-check-sigs --from "file:///tmp/nix-cache" "$path" + nix copy --no-check-sigs --from "file://${{ env.CACHED_PATH }}" "$path" done - name: Run `nix flake check` @@ -60,11 +62,11 @@ jobs: - name: Export cache from nix store if: steps.nix-store-cache.outputs.cache-hit != 'true' run: | - nix copy --to "file:///tmp/nix-cache?compression=zstd¶llel-compression=true" .#native-cargo-artifacts + nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#native-cargo-artifacts - uses: actions/cache/save@v3 # Write a cache entry even if the tests fail but don't create any for the merge queue. if: ${{ always() && steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: - path: /tmp/nix-cache + path: ${{ env.CACHED_PATH }} key: ${{ steps.cache.outputs.cache-primary-key }} From b4877832b5353cbeaa5b89cc7352056d864400c7 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 19:30:58 +0100 Subject: [PATCH 13/19] fix: apply same condition to moving from nix store as is for writing cache --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a7e7277d00..23a6716ab5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: nix flake check -L - name: Export cache from nix store - if: steps.nix-store-cache.outputs.cache-hit != 'true' + if: ${{ always() && steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} run: | nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#native-cargo-artifacts From 34f806194230e4893ac287f38e24712a5ccd80c3 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 19:56:32 +0100 Subject: [PATCH 14/19] chore: indentation --- .github/workflows/publish.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 12f57909e7..e8b80dfbb9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -154,11 +154,11 @@ jobs: env: CROSS_CONFIG: ${{ github.workspace }}/.github/Cross.toml CACHED_PATHS: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ strategy: fail-fast: false matrix: From d9379186ad0398546c4f5cdea37f95d2a2521e7a Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 20:00:58 +0100 Subject: [PATCH 15/19] chore: add cache to `build-wasm` job --- .github/workflows/wasm.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index f0c432515a..662a473e2d 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -104,6 +104,9 @@ jobs: build-wasm: runs-on: ubuntu-latest + env: + CACHED_PATH: /tmp/nix-cache + steps: - name: Checkout sources uses: actions/checkout@v3 @@ -114,10 +117,39 @@ jobs: nix_path: nixpkgs=channel:nixos-22.11 github_access_token: ${{ secrets.GITHUB_TOKEN }} + - name: Restore nix store cache + uses: actions/cache/restore@v3 + id: cache + with: + path: ${{ env.CACHED_PATH }} + key: ${{ runner.os }}-flake-wasm-${{ hashFiles('*.lock') }} + + # Based on https://github.com/marigold-dev/deku/blob/b5016f0cf4bf6ac48db9111b70dd7fb49b969dfd/.github/workflows/build.yml#L26 + - name: Copy cache into nix store + if: steps.cache.outputs.cache-hit == 'true' + # We don't check the signature because we're the one that created the cache + run: | + for narinfo in ${{ env.CACHED_PATH }}/*.narinfo; do + path=$(head -n 1 "$narinfo" | awk '{print $2}') + nix copy --no-check-sigs --from "file://${{ env.CACHED_PATH }}" "$path" + done + - name: Build wasm package run: | nix build -L .#wasm + - name: Export cache from nix store + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + run: | + nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#native-cargo-artifacts + + - uses: actions/cache/save@v3 + # Write a cache entry even if the tests fail but don't create any for the merge queue. + if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} + with: + path: ${{ env.CACHED_PATH }} + key: ${{ steps.cache.outputs.cache-primary-key }} + - name: Dereference symlink run: echo "UPLOAD_PATH=$(readlink -f result)" >> $GITHUB_ENV From af8f8b5a21e04f542d77a18ff51d263e96cd05a1 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 20:32:48 +0100 Subject: [PATCH 16/19] fix: correct command to save wasm artifacts --- .github/workflows/wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 662a473e2d..fb1104a95e 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -141,7 +141,7 @@ jobs: - name: Export cache from nix store if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} run: | - nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#native-cargo-artifacts + nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#wasm-cargo-artifacts - uses: actions/cache/save@v3 # Write a cache entry even if the tests fail but don't create any for the merge queue. From 1f7084ea45451a37b2ffe677e5af5ddb470c3eb5 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 20:51:31 +0100 Subject: [PATCH 17/19] chore: ok that's just confusing naming --- .github/workflows/wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index fb1104a95e..f3c8c8428d 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -141,7 +141,7 @@ jobs: - name: Export cache from nix store if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} run: | - nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#wasm-cargo-artifacts + nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#noir-wasm-cargo-artifacts - uses: actions/cache/save@v3 # Write a cache entry even if the tests fail but don't create any for the merge queue. From 09aa8db0af9d3d481b61a288105600ba51c1d37e Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 21:40:13 +0100 Subject: [PATCH 18/19] chore: expose `noir-wasm-cargo-artifacts` --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 9edae168a7..3c0af51946 100644 --- a/flake.nix +++ b/flake.nix @@ -254,6 +254,7 @@ # We expose the `*-cargo-artifacts` derivations so we can cache our cargo dependencies in CI inherit native-cargo-artifacts; inherit wasm-cargo-artifacts; + inherit noir-wasm-cargo-artifacts; }; # TODO(#1197): Look into installable apps with Nix flakes From a277f1566c92ef72933c4dd57111b8aa2a4b3315 Mon Sep 17 00:00:00 2001 From: Tom French Date: Wed, 16 Aug 2023 22:02:29 +0100 Subject: [PATCH 19/19] chore: update comment --- .github/workflows/wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index f3c8c8428d..f052e9eb07 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -144,7 +144,7 @@ jobs: nix copy --to "file://${{ env.CACHED_PATH }}?compression=zstd¶llel-compression=true" .#noir-wasm-cargo-artifacts - uses: actions/cache/save@v3 - # Write a cache entry even if the tests fail but don't create any for the merge queue. + # Don't create cache entries for the merge queue. if: ${{ steps.cache.outputs.cache-hit != 'true' && github.event_name != 'merge_group' }} with: path: ${{ env.CACHED_PATH }}