matrix foundry tests and snapshots (#10022)

* matrix foundry tests and snapshots * make snapshot great again * add product based make wrapper commands * update snapshots and reduce CI scope * use new wrapper gen in CI * split hardhat CI and generic Solidity CI make running HH CI more restrictive * add shared Foundry profile & fix typeAndVersion * clean up needs trigger and add comment * add foundry.toml CI trigger * getwrapper generation in products folders * rename llo -> llo-feeds * make snapshot-all * foundryup && update snapshots * fix naming in comments
smartcontractkit · Aug 15, 2023 · 67b1f2d · 67b1f2d
1 parent fb8116d
commit 67b1f2d
Show file tree

Hide file tree

Showing 16 changed files with 560 additions and 228 deletions.
diff --git a/.github/workflows/solidity-foundry.yml b/.github/workflows/solidity-foundry.yml
@@ -16,13 +16,20 @@ jobs:
       - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
         id: changes
         with:
+          # Foundry is only used for Solidity v0.8 contracts, therefore we can ignore
+          # changes to older contracts.
           filters: |
             src:
-              - 'contracts/**/*'
+              - 'contracts/src/v0.8/**/*'
               - '.github/workflows/solidity-foundry.yml'
+              - 'contacts/foundry.toml'
 
   tests:
+    strategy:
+      matrix:
+        product: [ vrf, automation, llo-feeds, functions, automation-dev, shared ]
     needs: [changes]
+    if: needs.changes.outputs.changes == 'true'
     name: Tests
     # See https://github.com/foundry-rs/foundry/issues/3827
     runs-on: ubuntu-22.04
@@ -37,42 +44,44 @@ jobs:
       # and not native Foundry. This is to make sure the dependencies
       # stay in sync.
       - name: Setup NodeJS
-        if: ${{ needs.changes.outputs.changes == 'true' }}
         uses: ./.github/actions/setup-nodejs
 
       - name: Install Foundry
-        if: ${{ needs.changes.outputs.changes == 'true' }}
         uses: foundry-rs/foundry-toolchain@v1
         with:
           version: nightly
 
       - name: Run Forge build
-        if: ${{ needs.changes.outputs.changes == 'true' }}
         run: |
           forge --version
           forge build
         id: build
         working-directory: contracts
+        env:
+          FOUNDRY_PROFILE: ${{ matrix.product }}
 
       - name: Run Forge tests
-        if: ${{ needs.changes.outputs.changes == 'true' }}
         run: |
           forge test -vvv
         id: test
         working-directory: contracts
+        env:
+          FOUNDRY_PROFILE: ${{ matrix.product }}
 
       - name: Run Forge snapshot
-        if: ${{ needs.changes.outputs.changes == 'true' }}
         run: |
-          forge snapshot --match-test _gas --check
+          forge snapshot --check gas-snapshots/${{ matrix.product }}.gas-snapshot
         id: snapshot
         working-directory: contracts
+        env:
+          FOUNDRY_PROFILE: ${{ matrix.product }}
+
       - name: Collect Metrics
         if: always()
         id: collect-gha-metrics
         uses: smartcontractkit/push-gha-metrics-action@6f41029bc7f0d0db5827f6804a2322e2a89e3b32
         with:
           basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
           hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
-          this-job-name: Foundry Tests
+          this-job-name: Foundry Tests ${{ matrix.product }}
         continue-on-error: true
diff --git a/.github/workflows/solidity-hardhat.yml b/.github/workflows/solidity-hardhat.yml
@@ -0,0 +1,177 @@
+name: Solidity-Hardhat
+
+on:
+  merge_group:
+  push:
+
+env:
+  NODE_OPTIONS: --max_old_space_size=8192
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      changes: ${{ steps.changes.outputs.src }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
+        id: changes
+        with:
+          filters: |
+            src:
+              - 'contracts/src/!(v0.8/(llo-feeds|ccip)/**)/**/*'
+              - 'contracts/test/**/*'
+              - 'contracts/package.json'
+              - 'contracts/hardhat.config.ts'
+              - 'contracts/ci.json'
+              - '.github/workflows/solidity-hardhat.yml'
+
+  split-tests:
+    name: Split Solidity Tests
+    runs-on: ubuntu-latest
+    outputs:
+      splits: ${{ steps.split.outputs.splits }}
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - name: Generate splits
+        id: split
+        uses: ./.github/actions/split-tests
+        with:
+          config: ./contracts/ci.json
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@6f41029bc7f0d0db5827f6804a2322e2a89e3b32
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: Split Solidity Tests
+        continue-on-error: true
+
+  solidity-coverage-splits:
+    needs: [changes, split-tests]
+    if: needs.changes.outputs.changes == 'true'
+    name: Solidity Coverage ${{ matrix.split.id }} ${{ fromJSON('["(skipped)", ""]')[needs.changes.outputs.changes == 'true'] }}
+    strategy:
+      fail-fast: false
+      matrix:
+        split: ${{ fromJson(needs.split-tests.outputs.splits) }}
+    runs-on: ubuntu20.04-4cores-16GB
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - name: Setup NodeJS
+        uses: ./.github/actions/setup-nodejs
+      - name: Setup Hardhat
+        uses: ./.github/actions/setup-hardhat
+        with:
+          namespace: coverage
+      - name: Run coverage
+        env:
+          SPLIT: ${{ matrix.split.coverageTests }}
+        shell: bash
+        run: pnpm coverage --testfiles "$SPLIT"
+        working-directory: contracts
+      - name: Push coverage
+        run: ./tools/bin/codecov -f ./contracts/coverage.json
+      - name: Rename coverage
+        run: mv ./contracts/coverage.json ./contracts/coverage-${{ matrix.split.idx }}.json
+      - name: Upload coverage
+        uses: actions/upload-artifact@v3
+        with:
+          name: solidity-coverage-${{ matrix.split.idx }}
+          path: ./contracts/coverage-${{ matrix.split.idx }}.json
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@6f41029bc7f0d0db5827f6804a2322e2a89e3b32
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: Solidity Coverage ${{ matrix.split.id }}
+        continue-on-error: true
+
+  solidity-coverage:
+    needs: [changes, solidity-coverage-splits]
+    if: needs.changes.outputs.changes == 'true'
+    name: Solidity Coverage ${{ fromJSON('["(skipped)", ""]')[needs.changes.outputs.changes == 'true'] }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - name: Setup NodeJS
+        uses: ./.github/actions/setup-nodejs
+      - name: Make coverage directory
+        run: mkdir ./contracts/coverage-reports
+      - name: Download coverage
+        uses: actions/download-artifact@v3
+        with:
+          path: ./contracts/coverage-reports
+      - name: Display structure of downloaded files
+        run: ls -R coverage-reports
+        working-directory: contracts
+      - name: Generate merged report
+        run: pnpm istanbul report text text-summary
+        working-directory: contracts
+
+  solidity-splits:
+    needs: [changes, split-tests]
+    if: needs.changes.outputs.changes == 'true'
+    name: Solidity ${{ matrix.split.id }} ${{ fromJSON('["(skipped)", ""]')[needs.changes.outputs.changes == 'true'] }}
+    strategy:
+      fail-fast: false
+      matrix:
+        split: ${{ fromJson(needs.split-tests.outputs.splits) }}
+    runs-on: ubuntu20.04-4cores-16GB
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - name: Setup NodeJS
+        uses: ./.github/actions/setup-nodejs
+      - name: Setup Hardhat
+        uses: ./.github/actions/setup-hardhat
+        with:
+          namespace: coverage
+      - name: Run tests
+        env:
+          SPLIT: ${{ matrix.split.tests }}
+        working-directory: contracts
+        run: pnpm test -- $SPLIT
+      - name: Collect Metrics
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@6f41029bc7f0d0db5827f6804a2322e2a89e3b32
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: Solidity ${{ matrix.split.id }}
+        continue-on-error: true
+
+  solidity:
+    needs: [changes, solidity-splits]
+    name: Solidity
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - run: echo 'Solidity tests finished!'
+      - name: Check test results
+        run: |
+          if [[ "${{ needs.changes.result }}" = "failure" || "${{ needs.solidity-splits.result }}" = "failure" ]]; then
+            echo "One or more changes / solidity-splits jobs failed"
+            exit 1
+          else
+            echo "All test jobs passed successfully"
+          fi
+      - name: Collect Metrics
+        if: always()
+        id: collect-gha-metrics
+        uses: smartcontractkit/push-gha-metrics-action@6f41029bc7f0d0db5827f6804a2322e2a89e3b32
+        with:
+          basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }}
+          hostname: ${{ secrets.GRAFANA_CLOUD_HOST }}
+          this-job-name: Solidity
+        continue-on-error: true