chore: custom cargo config to reduce build time

ChainSafe · Oct 20, 2022 · 4a2c4c4 · 4a2c4c4
1 parent 0f123ef
commit 4a2c4c4
Show file tree

Hide file tree

Showing 10 changed files with 123 additions and 77 deletions.
diff --git a/.cargo/config.toml b/.cargo/config.toml
@@ -0,0 +1,29 @@
+[alias]
+b = "build"
+c = "check"
+d = "doc"
+t = "test"
+r = "run"
+
+[build]
+incremental = true
+# might be helpful but we don't enforce sccache installation atm
+# https://github.com/mozilla/sccache
+# rustc-wrapper = "sccache"
+
+# For details checkout https://jondot.medium.com/8-steps-for-troubleshooting-your-rust-build-times-2ffc965fd13e
+[target.x86_64-unknown-linux-gnu]
+# clang has been listed as prerequisite in the doc
+linker    = "clang"
+rustflags = ["-Clink-arg=-fuse-ld=lld", "-Zshare-generics=y"]
+
+[target.x86_64-apple-darwin]
+# zld might help here
+# brew install michaeleisel/zld/zld
+# "-Clink-arg=-fuse-ld=zld"
+# For details checkout https://jondot.medium.com/8-steps-for-troubleshooting-your-rust-build-times-2ffc965fd13e
+rustflags = ["-Zshare-generics=y"]
+
+[net]
+git-fetch-with-cli = true
+retry              = 5
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -10,19 +10,18 @@ jobs:
     name: Cover
     runs-on: buildjet-8vcpu-ubuntu-2004
     timeout-minutes: 30
-    steps:
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential clang ocl-icd-opencl-dev
+    steps:      
       - name: Checkout Sources
         uses: actions/checkout@v3
         with:
           submodules: true
+      - name: Install Dependencies
+        run: sudo make install-deps
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: 5
-        continue-on-error: true
+        with:
+          timeout-minutes: 5
+          continue-on-error: true
       - name: Run cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - name: Generate code coverage

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -16,13 +16,11 @@ jobs:
   check-publish-docs:
     name: Publish
     runs-on: ubuntu-latest
-    steps:
-      - name: Apt Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential clang ocl-icd-opencl-dev
+    steps:      
       - name: Checkout Sources
         uses: actions/checkout@v3
+      - name: Apt Dependencies
+        run: sudo make install-deps
       - name: Link Checker (Repo)
         uses: lycheeverse/lychee-action@v1.5.1
         with:
@@ -31,8 +29,9 @@ jobs:
           fail: true
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: 5
-        continue-on-error: true
+        with:
+          timeout-minutes: 5
+          continue-on-error: true
       - name: Install MDBook
         run: cargo install mdbook mdbook-linkcheck
       - name: Execute MDBook

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -17,21 +17,20 @@ jobs:
           - os: macos-latest
             file: forest-${{ github.ref_name }}-macos-amd64.zip
     steps:
+      - name: Checkout Sources
+        uses: actions/checkout@v3
       - name: Apt Dependencies
         if: startsWith(matrix.os, 'Ubuntu')
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential clang ocl-icd-opencl-dev
+        run: sudo make install-deps
       - name: Homebrew Utils
         if: startsWith(matrix.os, 'macOS')
         run: |
           brew install --verbose coreutils
-      - name: Checkout Sources
-        uses: actions/checkout@v3
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: 5
-        continue-on-error: true
+        with:
+          timeout-minutes: 5
+          continue-on-error: true
       - name: Cargo Build
         run: cargo build --release --bin forest --bin forest-cli
       - name: Compress Binary

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -23,18 +23,17 @@ jobs:
     env:
       RUSTFLAGS: "-D warnings"
     steps:
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential clang ocl-icd-opencl-dev
       - name: Checkout Sources
         uses: actions/checkout@v3
+      - name: Install Dependencies
+        run: sudo make install-deps
       - name: install nextest
         uses: taiki-e/install-action@nextest
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
-        continue-on-error: true
+        with:
+          timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+          continue-on-error: true
       - name: Cargo Check
         run: cargo check
       - name: Make Test-All
@@ -44,16 +43,15 @@ jobs:
     name: Lint
     runs-on: ubuntu-latest
     steps:
-      - name: Apt Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential clang ocl-icd-opencl-dev
       - name: Checkout Sources
         uses: actions/checkout@v3
+      - name: Apt Dependencies
+        run: sudo make install-deps
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
-        continue-on-error: true
+        with:
+          timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+          continue-on-error: true
       - name: Install taplo (TOML linter)
         run: cargo install taplo-cli --locked
       - name: Run Linters
@@ -67,8 +65,9 @@ jobs:
         uses: actions/checkout@v3
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
-        continue-on-error: true
+        with:
+          timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+          continue-on-error: true
       - name: Install Audit
         run: cargo install cargo-audit
       - name: Run Audit
@@ -82,8 +81,9 @@ jobs:
         uses: actions/checkout@v3
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
-        continue-on-error: true
+        with:
+          timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+          continue-on-error: true
       - name: Install udeps
         run: cargo install cargo-udeps --locked
       - name: Run udeps
@@ -97,8 +97,9 @@ jobs:
         uses: actions/checkout@v3
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
-        continue-on-error: true
+        with:
+          timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+          continue-on-error: true
       - name: Install spellcheck
         run: cargo install cargo-spellcheck
       - name: Run Spellcheck
@@ -110,40 +111,52 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-        #rv: [1.58.1, stable, beta, nightly]
-        rv: [nightly]
     steps:
-      - name: Install Dependencies
-        if: startsWith(matrix.os, 'Ubuntu')
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential clang ocl-icd-opencl-dev
       - name: Checkout Sources
         uses: actions/checkout@v3
+      - uses: visvirial/sccache-action@v1
+        if: startsWith(matrix.os, 'Ubuntu')
+        with:
+          arch: x86_64-unknown-linux-musl
+      - uses: visvirial/sccache-action@v1
+        if: startsWith(matrix.os, 'macOS')
+        with:
+          arch: x86_64-apple-darwin
+      - name: Install Linux Dependencies
+        if: startsWith(matrix.os, 'Ubuntu')
+        run: |
+          lscpu # the job may run on different CPUs, list cpu here for analysing build time
+          sudo make install-deps
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
-        continue-on-error: true
+        with:
+          # Additional non workspace directories, separated by newlines
+          cache-directories: |
+            ~/.cache/sccache/
+          timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+          continue-on-error: true
       - name: Cargo Build
         run: cargo build --profile dev
+        env:
+          CC: "/tmp/sccache/sccache clang"
+          CXX: "/tmp/sccache/sccache clang++"
 
   calibnet-check:
     name: Calibnet sync check
     runs-on: buildjet-8vcpu-ubuntu-2004
     timeout-minutes: 30
     steps:
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y build-essential clang ocl-icd-opencl-dev
       - name: Checkout Sources
         uses: actions/checkout@v3
         with:
           submodules: true
+      - name: Install Dependencies
+        run: sudo make install-deps
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2
-        timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
-        continue-on-error: true
+        with:
+          timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}
+          continue-on-error: true
       - name: build and install binaries
         run: make install
       - name: download snapshot

diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,7 @@ config*.toml
 *.orig
 criterion
 genesis.json
-/.cargo
+/.cargo/*
+!/.cargo/*.toml
 node/rpc-api/static/ast.ron
 documentation/book/**/*
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Dockerfile b/Dockerfile
@@ -18,7 +18,7 @@
 FROM rust:1-buster AS build-env
 
 # Install dependencies
-RUN apt-get update && apt-get install --no-install-recommends -y build-essential clang ocl-icd-opencl-dev cmake
+RUN apt-get update && apt-get install --no-install-recommends -y build-essential clang lld ocl-icd-opencl-dev cmake
 
 WORKDIR /usr/src/forest
 COPY . .