fix(ci): significantly speed up CI. (#1124)

.dockerignore

@@ -1,4 +1,3 @@
 target
-fendermint/builtin-actors
 contracts/cache
 contracts/node-modules

.github/workflows/build.yaml

@@ -0,0 +1,128 @@
+name: Build IPC
+
+# This workflow is triggered from the main CI workflow.
+on:
+  workflow_call:
+
+jobs:
+  build:
+    name: Build IPC
+    runs-on: ubuntu-latest
+    env:
+      RUST_BACKTRACE: full
+      RUSTFLAGS: -Dwarnings
+
+    steps:
+      # https://github.com/marketplace/actions/free-disk-space-ubuntu
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          large-packages: false
+          swap-storage: false
+          docker-images: false
+          android: true
+          dotnet: true
+          haskell: true
+
+      - name: Check out the project
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install Tools
+        uses: ./.github/actions/install-tools
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          rust: stable
+
+      - name: Install Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: 18.19.0
+
+      - uses: Swatinem/rust-cache@v2
+        if: always()
+        with:
+          cache-on-failure: true
+
+      - name: Cache Solidity ABI artifacts
+        uses: actions/cache@v4
+        if: always()
+        with:
+          path: |
+            ./contracts/out
+            ./contracts/bindings
+            ./contracts/cache
+          ## TODO maybe add the rust version and solc version to the key
+          key: v2-contracts-abi-${{ hashFiles('./contracts/**/*.sol') }}
+
+      - name: Generate ABI and bindings
+        run: cd contracts && make gen
+
+      - name: Install Rust nightly
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: nightly
+          components: rustfmt,clippy
+
+      - name: Check fmt (fmt, clippy)
+        run: cargo +nightly fmt --check --all
+
+      - name: Check clippy
+        run: cargo clippy --tests --no-deps -- -D clippy::all
+
+      - name: Build all
+        run: |
+          cd fendermint && make $PWD/builtin-actors/output/bundle.car
+          cargo build --locked --release
+
+      - name: Build Docker image for e2e tests
+        run: |
+          ## Create the temporary Dockerfile.
+          cat <<EOF > /tmp/Dockerfile
+          # syntax=docker/dockerfile:1
+          FROM alpine as builder
+          COPY /fendermint/app/config /app/fendermint/app/config
+          COPY /target/release/fendermint /app/output/bin/fendermint
+          COPY /target/release/ipc-cli /app/output/bin/ipc-cli
+          EOF
+          
+          ## Append the runner build phase to the Dockerfile.
+          cat fendermint/docker/runner.Dockerfile >> /tmp/Dockerfile
+          
+          ## Print the Dockerfile for debugging.
+          echo "Dockerfile:"
+          cat /tmp/Dockerfile
+          
+          ## Create the temporary .dockerignore file.
+          cat <<EOF > /tmp/Dockerfile.dockerignore
+          target
+          !target/release/fendermint
+          !target/release/ipc-cli
+          contracts/cache
+          contracts/node-modules
+          EOF
+          
+          ## Print the .dockerignore file for debugging.
+          echo "Dockerfile.dockerignore:"
+          cat /tmp/Dockerfile.dockerignore
+          
+          ## Build the Docker image.
+          DOCKER_BUILDKIT=1 docker build \
+            --load \
+            -f /tmp/Dockerfile \
+            -t fendermint:latest \
+            .
+
+      - name: Create artifacts directory
+        run: mkdir -p /tmp/artifacts
+
+      - name: Export Docker image
+        run: docker save fendermint:latest > /tmp/artifacts/docker-image.tar
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image
+          path: /tmp/artifacts/docker-image.tar

.github/workflows/ci.yaml

@@ -23,53 +23,16 @@ on:
 
 
 jobs:
-  # JOB to run change detection
-  changes:
-    runs-on: ubuntu-latest
-    # Required permissions
-    permissions:
-      pull-requests: read
-    # Set job outputs to values from filter step
-    outputs:
-      workspace:     ${{ steps.filter.outputs.workspace }}
-      contracts:     ${{ steps.filter.outputs.contracts }}
-      ipc:           ${{ steps.filter.outputs.ipc }}
-      ipld-resolver: ${{ steps.filter.outputs.ipld-resolver }}
-      fendermint:    ${{ steps.filter.outputs.fendermint }}
-    steps:
-      # For pull requests it's not necessary to checkout the code,
-      # but the workflow is also triggered on pushes to `main`.
-      - uses: actions/checkout@v3
-
-      - uses: dorny/paths-filter@v2
-        id: filter
-        with:
-          filters: |
-            workspace:
-              - 'Cargo.toml'
-            contracts:
-              - 'contracts/**'
-            ipc:
-              - 'ipc/**'
-            ipld-resolver:
-              - 'ipld/resolver/**'
-            fendermint:
-              - 'fendermint/**'
 
   license:
     uses: ./.github/workflows/license.yaml
 
   contracts-prettier:
     uses: ./.github/workflows/contracts-prettier.yaml
-    needs: [changes]
-    if: >-
-      needs.changes.outputs.contracts == 'true' ||
-      github.ref == 'refs/heads/main' ||
-      github.ref_type == 'tag'
 
   contracts-deployment-test:
     uses: ./.github/workflows/contracts-deployment-test.yaml
-    needs: [ contracts-prettier ]
+    needs: [contracts-prettier]
 
   contracts-test:
     uses: ./.github/workflows/contracts-test.yaml
@@ -79,9 +42,12 @@ jobs:
     uses: ./.github/workflows/contracts-storage.yaml
     needs: [contracts-prettier]
 
-  contracts-pnpm-audit:
-    uses: ./.github/workflows/contracts-pnpm-audit.yaml
-    needs: [contracts-prettier]
+## This causes too much unpredictable noise, turning our builds red whenever a new vulnerability is found.
+## It's better to have a separate workflow for auditing, and have it run on a schedule.
+## However, given this code is purely used for tooling and not for production, it's not a big deal.
+#  contracts-pnpm-audit:
+#    uses: ./.github/workflows/contracts-pnpm-audit.yaml
+#    needs: [contracts-prettier]
 
   contracts-sast:
     uses: ./.github/workflows/contracts-sast.yaml
@@ -91,53 +57,33 @@ jobs:
     uses: ./.github/workflows/extras.yaml
     needs: [contracts-prettier]
 
-  ipc:
-    uses: ./.github/workflows/ipc.yaml
-    needs: [changes, license]
-    if: >-
-      needs.changes.outputs.workspace == 'true' ||
-      needs.changes.outputs.contracts == 'true' ||
-      needs.changes.outputs.ipc == 'true' ||
-      github.ref == 'refs/heads/main' ||
-      github.ref_type == 'tag'
+  build:
+    uses: ./.github/workflows/build.yaml
+    needs: [license]
 
-  ipld-resolver:
-    uses: ./.github/workflows/ipld-resolver.yaml
-    needs: [changes, license]
-    if: >-
-      needs.changes.outputs.workspace == 'true' ||
-      needs.changes.outputs.ipld-resolver == 'true' ||
-      github.ref == 'refs/heads/main' ||
-      github.ref_type == 'tag'
+  unit-tests:
+    uses: ./.github/workflows/tests-unit.yaml
+    secrets: inherit
+    needs:
+      - build
 
-  fendermint-test:
-    uses: ./.github/workflows/fendermint-test.yaml
+  e2e-tests:
+    uses: ./.github/workflows/tests-e2e.yaml
     secrets: inherit
-    needs: [changes, license]
-    if: >-
-      needs.changes.outputs.workspace == 'true' ||
-      needs.changes.outputs.contracts == 'true' ||
-      needs.changes.outputs.ipc == 'true' ||
-      needs.changes.outputs.ipld-resolver == 'true' ||
-      needs.changes.outputs.fendermint == 'true' ||
-      github.ref == 'refs/heads/main' ||
-      github.ref_type == 'tag'
+    needs:
+      - build
 
   fevm-contract-tests:
     uses: ./.github/workflows/fevm-contract-tests.yaml
     secrets: inherit
-    needs: [changes, license]
+    needs:
+      - build
     if: >-
-      needs.changes.outputs.workspace == 'true' ||
-      needs.changes.outputs.contracts == 'true' ||
-      needs.changes.outputs.ipc == 'true' ||
-      needs.changes.outputs.ipld-resolver == 'true' ||
-      needs.changes.outputs.fendermint == 'true' ||
       github.ref == 'refs/heads/main' ||
       github.ref_type == 'tag'
 
-  fendermint-publish:
-    uses: ./.github/workflows/fendermint-publish.yaml
+  docker-publish:
+    uses: ./.github/workflows/docker-publish.yaml
     secrets: inherit
     # Dependencies are not strictly necessary, but if fendermint tests pass they publish docker too, so they better work.
     # It is because of these needs that all the filters are allowed to run on `main` too, otherwise this would be disabled.
@@ -148,6 +94,6 @@ jobs:
       github.ref_type == 'tag'
     needs:
       - contracts-test # generates the ABI artifacts (although fendermint can do on its own too)
-      - ipc
-      - ipld-resolver
-      - fendermint-test
+      - build
+      - unit-tests
+      - e2e-tests

.github/workflows/contracts-deployment-test.yaml

@@ -1,4 +1,4 @@
-name: Deployment Test
+name: 'Contracts: Smoke test deployment'
 
 # This workflow is triggered from the main CI workflow.
 on:

.github/workflows/contracts-pnpm-audit.yaml

@@ -1,4 +1,4 @@
-name: NPM Audit
+name: 'Contracts: pnpm audit'
 
 # This workflow is triggered from the main CI workflow.
 on:

.github/workflows/contracts-prettier.yaml

@@ -1,5 +1,5 @@
-# .github/workflows/prettier.yml
-name: Prettier
+
+name: 'Contracts: Prettier'
 
 # This workflow is triggered from the main CI workflow.
 on:
@@ -8,30 +8,23 @@ on:
 jobs:
   prettier:
     runs-on: ubuntu-latest
-    permissions:
-      # Give the default GITHUB_TOKEN write permission to commit and push the
-      # added or changed files to the repository.
-      contents: write
     steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.ref }}
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - uses: pnpm/action-setup@v2
+
       - name: Set up node.js
         uses: actions/setup-node@v4
         with:
           node-version: '21'
           cache: 'pnpm'
+
       - name: Run formatter
         run: cd contracts && make fmt
+
       - name: Check diff clean
         run: |
           git status
           git diff --name-only
           git --no-pager diff --quiet
-
-#      - name: Commit changes
-#        uses: stefanzweifel/git-auto-commit-action@v5
-#        with:
-#          commit_message: Apply Prettier formatting
-#          branch: ${{ github.head_ref }}

.github/workflows/contracts-sast.yaml

@@ -1,4 +1,4 @@
-name: Static analysis
+name: 'Contracts: Static analysis'
 
 # This workflow is triggered from the main CI workflow.
 on:

.github/workflows/contracts-storage.yaml

@@ -1,4 +1,4 @@
-name: Storage check
+name: 'Contracts: Storage check'
 
 # This workflow is triggered from the main CI workflow.
 on: