Speed up CI: less use of ci_docker (#5896)

### What * Use less of `ci_docker` (it takes a minute to start!) * Smaller tweaks Saves around 5min of billable time (1h20m -> 1h15m) ### Checklist * [x] I've checked this checkbox - [PR Build Summary](https://build.rerun.io/pr/5896) - [Recent benchmark results](https://build.rerun.io/graphs/crates.html) - [Wasm size tracking](https://build.rerun.io/graphs/sizes.html)
rerun-io · Apr 10, 2024 · bc1becf · bc1becf
1 parent 62ff2d5
commit bc1becf
Show file tree

Hide file tree

Showing 12 changed files with 36 additions and 48 deletions.
diff --git a/.github/workflows/contrib_checks.yml b/.github/workflows/contrib_checks.yml
@@ -84,20 +84,12 @@ jobs:
   no-codegen-changes:
     name: Check if running codegen would produce any changes
     runs-on: ubuntu-latest-16-cores
-    container:
-      image: rerunio/ci_docker:0.12.0
     steps:
       # Note: We explicitly don't override `ref` here. We need to see if changes would be made
       # in a context where we have merged with main. Otherwise we might miss changes such as one
       # PR introduces a new type and another PR changes the codegen.
       - uses: actions/checkout@v4
 
-      # So we can format the Python code
-      - name: Set up Python
-        shell: bash
-        run: |
-          pip install -r rerun_py/requirements-lint.txt
-
       - uses: prefix-dev/setup-pixi@v0.4.1
         with:
           pixi-version: v0.18.0
@@ -181,8 +173,6 @@ jobs:
   rs-check-wasm:
     name: Check Rust web build (wasm32 + wasm-bindgen)
     runs-on: ubuntu-latest-16-cores
-    container:
-      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
@@ -258,11 +248,12 @@ jobs:
   rs-cargo-deny:
     name: Cargo Deny
     runs-on: ubuntu-latest
-    container:
-      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
 
+      - name: Install cargo-deny
+        run: cargo install --locked --quiet cargo-deny
+
       - name: Cargo Deny
         shell: bash
         id: expected_version

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -39,7 +39,7 @@ jobs:
     secrets: inherit
 
   build-web:
-    name: "Build Web"
+    name: "Build and upload web viewer"
     uses: ./.github/workflows/reusable_build_web.yml
     with:
       CONCURRENCY: nightly

diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -149,15 +149,15 @@ jobs:
     secrets: inherit
 
   build-js:
-    name: "Build JS"
+    name: "Build rerun_js"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
     uses: ./.github/workflows/reusable_build_js.yml
     with:
       CONCURRENCY: pr-${{ github.event.pull_request.number }}
     secrets: inherit
 
   build-web:
-    name: "Build Web"
+    name: "Build and upload web viewer"
     if: github.event.pull_request.head.repo.owner.login == 'rerun-io'
     uses: ./.github/workflows/reusable_build_web.yml
     with:

diff --git a/.github/workflows/on_push_main.yml b/.github/workflows/on_push_main.yml
@@ -77,7 +77,7 @@ jobs:
     secrets: inherit
 
   build-web:
-    name: "Build Web"
+    name: "Build and upload web viewer"
     uses: ./.github/workflows/reusable_build_web.yml
     with:
       CONCURRENCY: push-${{ github.ref_name }}

diff --git a/.github/workflows/reusable_build_and_upload_wheels.yml b/.github/workflows/reusable_build_and_upload_wheels.yml
@@ -101,14 +101,14 @@ jobs:
             linux-arm64)
                 runner="buildjet-8vcpu-ubuntu-2204-arm"
                 target="aarch64-unknown-linux-gnu"
-                container="'rerunio/ci_docker:0.12.0'"
+                container="'rerunio/ci_docker:0.12.0'" # Required to be manylinux compatible
                 compat="manylinux_2_31"
                 ;;
             linux-x64)
               runner="ubuntu-latest-16-cores"
               target="x86_64-unknown-linux-gnu"
               compat="manylinux_2_31"
-              container="'rerunio/ci_docker:0.12.0'"
+              container="'rerunio/ci_docker:0.12.0'" # Required to be manylinux compatible
               ;;
             windows-x64)
               runner="windows-latest-8-cores"

diff --git a/.github/workflows/reusable_build_js.yml b/.github/workflows/reusable_build_js.yml
@@ -1,4 +1,4 @@
-name: Reusable Build JS
+name: Reusable Build rerun_js
 
 on:
   workflow_call:
@@ -22,16 +22,13 @@ env:
 
 jobs:
   build:
-    name: Build
+    name: Build rerun_js
     permissions:
       contents: "read"
       id-token: "write"
 
     runs-on: ubuntu-latest-16-cores
 
-    container:
-      image: rerunio/ci_docker:0.12.0
-
     steps:
       - uses: actions/checkout@v4
         with:
@@ -57,10 +54,10 @@ jobs:
         with:
           pixi-version: v0.18.0
 
-      - name: Install dependencies
+      - name: Install yarn dependencies
         shell: bash
         run: pixi run yarn --cwd rerun_js install
 
-      - name: Build package
+      - name: Build rerun_js package
         shell: bash
         run: pixi run yarn --cwd rerun_js workspaces run build
diff --git a/.github/workflows/reusable_build_web.yml b/.github/workflows/reusable_build_web.yml
@@ -1,4 +1,4 @@
-name: Reusable Build Web
+name: Reusable Build and upload web viewer
 
 on:
   workflow_call:
@@ -29,16 +29,13 @@ env:
 
 jobs:
   rs-build-web-viewer:
-    name: Build Web (wasm32 + wasm-bindgen)
+    name: Build and upload web viewer
     permissions:
       contents: "read"
       id-token: "write"
 
     runs-on: ubuntu-latest-16-cores
 
-    container:
-      image: rerunio/ci_docker:0.12.0
-
     steps:
       - uses: actions/checkout@v4
         with:
@@ -78,7 +75,7 @@ jobs:
             --channel "${{ inputs.CHANNEL }}" \
             web_viewer/examples_manifest.json
 
-      - name: Upload web assets
+      - name: Upload web viewer
         uses: actions/upload-artifact@v3
         with:
           name: web_viewer

diff --git a/.github/workflows/reusable_checks.yml b/.github/workflows/reusable_checks.yml
@@ -82,8 +82,6 @@ jobs:
   no-codegen-changes:
     name: Check if running codegen would produce any changes
     runs-on: ubuntu-latest-16-cores
-    container:
-      image: rerunio/ci_docker:0.12.0
     env:
       RUSTC_WRAPPER: "sccache"
     steps:
@@ -100,12 +98,6 @@ jobs:
           workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
           service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
 
-      # So we can format the Python code
-      - name: Set up Python
-        shell: bash
-        run: |
-          pip install -r rerun_py/requirements-lint.txt
-
       - uses: prefix-dev/setup-pixi@v0.4.1
         with:
           pixi-version: v0.18.0

diff --git a/.github/workflows/reusable_checks_rust.yml b/.github/workflows/reusable_checks_rust.yml
@@ -38,8 +38,6 @@ jobs:
   rs-lints:
     name: Rust lints (fmt, check, cranky, tests, doc)
     runs-on: ubuntu-latest-16-cores
-    container:
-      image: rerunio/ci_docker:0.12.0
     env:
       RUSTC_WRAPPER: "sccache"
     steps:
@@ -66,6 +64,9 @@ jobs:
         run: |
           pixi run cargo run --locked -p re_build_web_viewer -- --release -g
 
+      - name: install cargo cranky
+        run: cargo install --locked --quiet cargo-cranky
+
       - name: Rust checks & tests
         if: ${{ !inputs.ALL_CHECKS }}
         run: ./scripts/ci/rust_checks.py --skip-check-individual-crates
@@ -79,8 +80,6 @@ jobs:
   rs-check-wasm:
     name: Check Rust web build (wasm32 + wasm-bindgen)
     runs-on: ubuntu-latest-16-cores
-    container:
-      image: rerunio/ci_docker:0.12.0
     env:
       RUSTC_WRAPPER: "sccache"
     steps:
@@ -121,13 +120,22 @@ jobs:
   rs-cargo-deny:
     name: Cargo Deny
     runs-on: ubuntu-latest
-    container:
-      image: rerunio/ci_docker:0.12.0
     steps:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || '' }}
 
+      - name: Set up Rust
+        uses: ./.github/actions/setup-rust
+        with:
+          cache_key: "rs-cargo-deny"
+          save_cache: true
+          workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }}
+          service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }}
+
+      - name: Install cargo-deny
+        run: cargo install --locked --quiet cargo-deny
+
       - name: Cargo Deny
         shell: bash
         id: expected_version

diff --git a/pixi.toml b/pixi.toml
@@ -28,7 +28,7 @@ cpp = ["cpp"]
 # Note: extra CLI argument after `pixi run TASK` are passed to the task cmd.
 
 # Run the codegen. Optionally pass `--profile` argument if you want.
-codegen = "cargo run --package re_types_builder -- "
+codegen = "cargo --quiet run --package re_types_builder -- "
 
 # Compile and run the rerun viewer.
 #
@@ -86,7 +86,7 @@ fmt = { depends_on = ["format"] }
 
 # Assorted linting tasks
 fast-lint = "python scripts/fast_lint.py"
-lint-codegen = "cargo run --package re_types_builder -- --check"
+lint-codegen = "cargo --quiet run --package re_types_builder -- --check"
 # TODO(jleibs): implement lint-cpp-all
 lint-rerun = "python scripts/lint.py"
 lint-rs-files = "rustfmt --edition 2021 --check"

diff --git a/scripts/ci/cargo_deny.sh b/scripts/ci/cargo_deny.sh
@@ -5,7 +5,8 @@ script_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 cd "$script_path/../.."
 set -x
 
-# cargo install cargo-deny
+cargo install --locked --quiet cargo-deny
+
 cargo deny --all-features --log-level error --target aarch64-apple-darwin check
 cargo deny --all-features --log-level error --target i686-pc-windows-gnu check
 cargo deny --all-features --log-level error --target i686-pc-windows-msvc check

diff --git a/scripts/clippy_wasm.sh b/scripts/clippy_wasm.sh
@@ -10,4 +10,6 @@ set -x
 # Use scripts/clippy_wasm/clippy.toml
 export CLIPPY_CONF_DIR="scripts/clippy_wasm"
 
+cargo install --locked --quiet cargo-cranky
+
 cargo cranky --all-features --target wasm32-unknown-unknown --target-dir target_wasm -p re_viewer -- --deny warnings