ci: use prebuilt librocksdb in github actions (#2316)

.github/actions/librocksdb/action.yaml

@@ -0,0 +1,55 @@
+---
+# This action builds and caches librocksdb. If we find that this solution consumes too much time, we can consider
+# prebuilding librocksdb outside of the pipeline (eg. in the grovedb release process), publish as an artifact, and
+# download it in the pipeline.
+name: "librocksdb"
+description: "Build and install librocksdb"
+inputs:
+  version:
+    description: RocksDB version, eg. "8.10.2"
+    required: false
+    default: "8.10.2"
+  bucket:
+    description: S3 bucket to use for caching
+    required: false
+    default: multi-runner-cache-x1xibo9c
+  force:
+    description: Force rebuild
+    required: false
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+    # Cache librocksdb using s3 bucket
+    - name: Restore cached librocksdb from S3
+      id: librocksdb-cache
+      uses: strophy/actions-cache@opendal-update
+      with:
+        bucket: ${{ inputs.bucket }}
+        path: /opt/rocksdb
+        key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
+
+    - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' || inputs.force == 'true' }}
+      shell: bash
+      name: Build librocksdb
+      run: |
+        set -ex
+        WORKDIR=/tmp/rocksdb-build
+        mkdir -p ${WORKDIR}/rocksdb
+        mkdir -p /opt/rocksdb/usr/local/lib/
+        pushd ${WORKDIR}/rocksdb
+
+        # building rocksdb
+        git clone https://github.com/facebook/rocksdb.git -b v${{ inputs.version }} --depth 1 .
+        make -j$(nproc) static_lib
+        make DESTDIR=/opt/rocksdb install-static
+        set +x
+
+        echo Done.
+        echo Configuration:
+        echo
+        echo "ROCKSDB_STATIC='/opt/rocksdb/usr/local/lib/librocksdb.a'"
+        echo "ROCKSDB_LIB_DIR='/opt/rocksdb/usr/local/lib'"
+
+        popd

.github/workflows/cached.yml

@@ -0,0 +1,23 @@
+---
+name: Rebuild cached dependencies
+
+on:
+  workflow_dispatch:
+jobs:
+  build-rust-deps:
+    name: Prebuild and cache some Rust dependencies
+    runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Precompile librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          force: true

.github/workflows/docs.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   build:
     name: Deploy docs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout main
         uses: actions/checkout@v4

.github/workflows/manage-runs.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   cancel-merged-or-closed-pr-runs:
     name: Cancel runs for merged or closed PRs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: octokit/request-action@v2.x
         id: get_active_workflows

.github/workflows/pr.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   pr-title:
     name: PR title
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Validate conventional PR title
         uses: amannn/action-semantic-pull-request@v5

.github/workflows/release-docker-image.yml

@@ -89,7 +89,7 @@ jobs:
   publish-manifest:
     name: Publish image tags
     needs: build-image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Download digests
         uses: actions/download-artifact@v3

.github/workflows/release.yml

@@ -208,11 +208,11 @@ jobs:
       matrix:
         include:
           - package_type: tarballs
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: win
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: deb
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: macos
             os: macos-14
     steps:

.github/workflows/tests-codeql.yml

@@ -4,7 +4,7 @@ on:
 jobs:
   codeql:
     name: Run Code QL
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       actions: read
       contents: read

.github/workflows/tests-js-package.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   lint:
     name: Linting
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read
@@ -51,7 +51,7 @@ jobs:
 
   test:
     name: Tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read

.github/workflows/tests-rs-package.yml

@@ -12,7 +12,7 @@ on:
       lint-runner:
         description: Runner for linting. Must be JSON valid string.
         type: string
-        default: '"ubuntu-22.04"'
+        default: '"ubuntu-24.04"'
       check-each-feature:
         description: If true, try to build each individual feature for this crate
         type: boolean
@@ -42,6 +42,9 @@ jobs:
         with:
           components: clippy
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+
       - uses: clechasseur/rs-clippy-check@v3
         with:
           args: --package ${{ inputs.package }} --all-features --locked -- --no-deps
@@ -50,10 +53,12 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
 
   formatting:
     name: Formatting
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     timeout-minutes: 5
     steps:
       - name: Check out repo
@@ -65,12 +70,19 @@ jobs:
           components: rustfmt
           cache: false
 
+      #  This step doesn't need librocksdb, so we don't install it
+
       - name: Check formatting
+        env:
+          RUSTC_WRAPPER: sccache
+          SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
+          SCCACHE_REGION: ${{ secrets.AWS_REGION }}
+          SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
         run: cargo fmt --check --package=${{ inputs.package }}
 
   unused_deps:
     name: Unused dependencies
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read
@@ -89,6 +101,9 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+
       - name: Get crate ${{ inputs.package }} info
         id: crate_info
         uses: ./.github/actions/crate_info
@@ -102,12 +117,14 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
         with:
           args: ${{ steps.crate_info.outputs.cargo_manifest_dir }}
 
   detect_structure_changes:
     name: Detect immutable structure changes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     # FIXME: as we use `gh pr view` below, this check can only
     # run on pull requests. We should find a way to run it
     # when manual triggers are used.
@@ -184,13 +201,18 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+
       - name: Run tests
         run: cargo test --package=${{ inputs.package }} --all-features --locked
         env:
           RUSTC_WRAPPER: sccache
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
 
   check_each_feature:
     name: Check each feature
@@ -199,7 +221,7 @@ jobs:
     if: ${{ inputs.check-each-feature }}
     steps:
       - name: Check out repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Configure AWS credentials and bucket region
         uses: aws-actions/configure-aws-credentials@v4
@@ -211,6 +233,9 @@ jobs:
       - name: Setup Rust
         uses: ./.github/actions/rust
 
+      - name: Install librocksdb
+        uses: ./.github/actions/librocksdb
+
       - name: Get crate ${{ runner.arch }} info
         id: crate_info
         uses: ./.github/actions/crate_info
@@ -223,6 +248,8 @@ jobs:
           SCCACHE_BUCKET: multi-runner-cache-x1xibo9c
           SCCACHE_REGION: ${{ secrets.AWS_REGION }}
           SCCACHE_S3_KEY_PREFIX: ${{ runner.os }}/sccache/${{ runner.arch }}/linux-gnu
+          ROCKSDB_STATIC: "/opt/rocksdb/usr/local/lib/librocksdb.a"
+          ROCKSDB_LIB_DIR: "/opt/rocksdb/usr/local/lib"
         run: |
           echo Verify all features disabled
           set -ex

.github/workflows/tests.yml

@@ -2,6 +2,11 @@ name: Tests
 
 on:
   workflow_dispatch:
+    inputs:
+      rebuild-deps:
+        description: "Rebuild cached Rust dependencies"
+        required: false
+        default: "false"
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
     branches:
@@ -16,13 +21,13 @@ jobs:
   changes:
     name: Determine changed packages
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     outputs:
       js-packages: ${{ steps.filter-js.outputs.changes }}
       rs-packages: ${{ steps.filter-rs.outputs.changes }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -36,6 +41,22 @@ jobs:
         with:
           filters: .github/package-filters/rs-packages.yml
 
+  build-rust-deps:
+    name: Prebuild and cache some Rust dependencies
+    runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Precompile librocksdb
+        uses: ./.github/actions/librocksdb
+
   build-js:
     name: Build JS packages
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
@@ -69,6 +90,7 @@ jobs:
     name: Rust packages
     needs:
       - changes
+      - build-rust-deps
     secrets: inherit
     strategy:
       fail-fast: false
@@ -77,17 +99,19 @@ jobs:
     uses: ./.github/workflows/tests-rs-package.yml
     with:
       package: ${{ matrix.rs-package }}
-      # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-22.04"' }}
+      # lint-runner: ${{ contains(fromJSON('["drive-abci", "drive"]'), matrix.rs-package) && '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]' || '"ubuntu-24.04"' }}
       # FIXME: Clippy fails on github hosted runners, most likely due to RAM usage. Using self-hosted runners for now.
-      lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      # lint-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      lint-runner: '["ubuntu-24.04"]'
       # Run drive tests on self-hosted 4x
-      test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      # test-runner: '[ "self-hosted", "linux", "arm64", "ubuntu-platform" ]'
+      test-runner: '["ubuntu-24.04"]'
       check-each-feature: ${{ contains(fromJSON('["dash-sdk","rs-dapi-client","dapi-grpc","dpp","drive-abci"]'), matrix.rs-package) }}
 
   rs-crates-security:
     name: Rust crates security audit
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -116,7 +140,7 @@ jobs:
   js-deps-versions:
     name: JS dependency versions check
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -135,7 +159,7 @@ jobs:
   js-npm-security:
     name: JS NPM security audit
     if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || !github.event.pull_request.draft }}
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4