ci: switch to free runners

.github/actions/docker/action.yaml

@@ -69,12 +69,12 @@ runs:
           [worker.oci]
             gc = false
 
-    - name: Load Docker mount cache
-      uses: dcginfra/buildkit-cache-dance/inject@s5cmd
-      if: ${{ inputs.cache_mounts != '' }}
-      with:
-        bucket: ${{ inputs.bucket }}
-        mounts: ${{ inputs.cache_mounts }}
+#    - name: Load Docker mount cache
+#      uses: dcginfra/buildkit-cache-dance/inject@s5cmd
+#      if: ${{ inputs.cache_mounts != '' }}
+#      with:
+#        bucket: ${{ inputs.bucket }}
+#        mounts: ${{ inputs.cache_mounts }}
 
     - name: Set Docker tags and labels from image tag
       id: docker_meta
@@ -85,21 +85,79 @@ runs:
           type=semver,pattern={{version}},value=${{inputs.image_version}},enable=${{ inputs.image_version != '' }}
           type=sha,format=long,enable=${{ inputs.image_version == '' }}
 
-    - name: Configure docker layer cache
-      uses: ./.github/actions/s3-layer-cache-settings
-      id: layer_cache_settings
+#    - name: Configure docker layer cache
+#      uses: ./.github/actions/s3-layer-cache-settings
+#      id: layer_cache_settings
+#      with:
+#        name: ${{ inputs.image_name }}
+#        bucket: ${{ inputs.bucket }}
+
+    RUN --mount=type=cache,sharing=shared,id=,target=${CARGO_HOME}/registry/index \
+    --mount=type=cache,sharing=shared,id=cargo_registry_cache,target=${CARGO_HOME}/registry/cache \
+    --mount=type=cache,sharing=shared,id=cargo_git,target=${CARGO_HOME}/git/db \
+
+    - name: Cargo cache for Docker
+      uses: actions/cache@v4
+      id: cargo-cache
+      with:
+        path: |
+          ../cargo-cache-registry-index
+          ../cargo-cache-registry-cache
+          ../cargo-cache-git-db
+        key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
+        restore-keys:
+            - ${{ runner.os }}-cargo-
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/buildkit-cache-dance@3
       with:
-        name: ${{ inputs.image_name }}
-        bucket: ${{ inputs.bucket }}
+        cache-map: |
+          {
+            "../cargo-cache-registry-index": {
+              "target": "/root/.cargo/registry/index",
+              "id": "cargo_registry_index"
+            },
+            "../cargo-cache-registry-cache": {
+              "target": "/root/.cargo/registry/cache",
+              "id": "cargo_registry_index"
+            },
+            "../cargo-cache-git-db": {
+              "target": "/root/.cargo/git/db",
+              "id": "cargo_registry_index"
+            },
+          }
+        skip-extraction: ${{ steps.cargo-cache.outputs.cache-hit }}
+
+    - name: Yarn unplugged cache for Docker
+      uses: actions/cache@v4
+      id: yarn-cache
+      with:
+        path: ../yarn-unplugged-cache
+        key: ${{ runner.os }}-yarn-unplugged-amd64-${{ hashFiles('yarn.lock') }}
+        restore-keys:
+          - ${{ runner.os }}-yarn-unplugged-amd64-
+
+    - name: Inject cargo cache into docker
+      uses: reproducible-containers/buildkit-cache-dance@3
+      with:
+        cache-map: |
+          {
+            "../yarn-unplugged-cache": {
+              "target": "/tmp/unplugged",
+              "id": "unplugged_amd64"
+            },
+          }
+        skip-extraction: ${{ steps.yarn-cache.outputs.cache-hit }}
 
     - name: Build and push Docker image ${{ inputs.image }}
       id: docker_build
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       with:
         context: .
         builder: ${{ steps.buildx.outputs.name }}
         target: ${{ inputs.target }}
         labels: ${{ steps.docker_meta.outputs.labels }}
+        push: ${{ inputs.push_tags }}
         tags: ${{ inputs.push_tags == 'true' && steps.docker_meta.outputs.tags || '' }}
         platforms: ${{ inputs.platform }}
         build-args: |
@@ -110,13 +168,15 @@ runs:
           SCCACHE_S3_KEY_PREFIX=${{ runner.os }}/sccache
           AWS_ACCESS_KEY_ID=${{ inputs.aws_access_key_id }}
           AWS_SECRET_ACCESS_KEY=${{ inputs.aws_secret_access_key }}
-        cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
-        cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
+        #cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
+        cache-from: type=gha
+        #cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}
+        cache-to: type=gha,mode=max
         outputs: type=image,name=${{ inputs.image_org }}/${{ inputs.image_name }},push-by-digest=${{ inputs.push_tags != 'true' }},name-canonical=true,push=true
 
-    - name: Save Docker mount cache
-      uses: dcginfra/buildkit-cache-dance/extract@s5cmd
-      if: ${{ inputs.cache_mounts != '' }}
-      with:
-        bucket: ${{ inputs.bucket }}
-        mounts: ${{ inputs.cache_mounts }}
+#    - name: Save Docker mount cache
+#      uses: dcginfra/buildkit-cache-dance/extract@s5cmd
+#      if: ${{ inputs.cache_mounts != '' }}
+#      with:
+#        bucket: ${{ inputs.bucket }}
+#        mounts: ${{ inputs.cache_mounts }}

.github/actions/librocksdb/action.yaml

@@ -0,0 +1,55 @@
+---
+# This action builds and caches librocksdb. If we find that this solution consumes too much time, we can consider
+# prebuilding librocksdb outside of the pipeline (eg. in the grovedb release process), publish as an artifact, and
+# download it in the pipeline.
+name: "librocksdb"
+description: "Build and install librocksdb"
+inputs:
+  version:
+    description: RocksDB version, eg. "8.10.2"
+    required: false
+    default: "8.10.2"
+  bucket:
+    description: S3 bucket to use for caching
+    required: false
+    default: multi-runner-cache-x1xibo9c
+  force:
+    description: Force rebuild
+    required: false
+    default: "false"
+
+runs:
+  using: composite
+  steps:
+    # Cache librocksdb using s3 bucket
+    - name: Restore cached librocksdb from S3
+      id: librocksdb-cache
+      uses: strophy/actions-cache@opendal-update
+      with:
+        bucket: ${{ inputs.bucket }}
+        path: /opt/rocksdb
+        key: librocksdb/${{ inputs.version }}/${{ runner.os }}/${{ runner.arch }}
+
+    - if: ${{ steps.librocksdb-cache.outputs.cache-hit != 'true' || inputs.force == 'true' }}
+      shell: bash
+      name: Build librocksdb
+      run: |
+        set -ex
+        WORKDIR=/tmp/rocksdb-build
+        mkdir -p ${WORKDIR}/rocksdb
+        mkdir -p /opt/rocksdb/usr/local/lib/
+        pushd ${WORKDIR}/rocksdb
+
+        # building rocksdb
+        git clone https://github.com/facebook/rocksdb.git -b v${{ inputs.version }} --depth 1 .
+        make -j$(nproc) static_lib
+        make DESTDIR=/opt/rocksdb install-static
+        set +x
+
+        echo Done.
+        echo Configuration:
+        echo
+        echo "ROCKSDB_STATIC='/opt/rocksdb/usr/local/lib/librocksdb.a'"
+        echo "ROCKSDB_LIB_DIR='/opt/rocksdb/usr/local/lib'"
+
+        popd

.github/workflows/cached.yml

@@ -0,0 +1,23 @@
+---
+name: Rebuild cached dependencies
+
+on:
+  workflow_dispatch:
+jobs:
+  build-rust-deps:
+    name: Prebuild and cache some Rust dependencies
+    runs-on: ubuntu-24.04
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+      - name: Precompile librocksdb
+        uses: ./.github/actions/librocksdb
+        with:
+          force: true

.github/workflows/docs.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   build:
     name: Deploy docs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout main
         uses: actions/checkout@v4

.github/workflows/manage-runs.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   cancel-merged-or-closed-pr-runs:
     name: Cancel runs for merged or closed PRs
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: octokit/request-action@v2.x
         id: get_active_workflows

.github/workflows/pr.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   pr-title:
     name: PR title
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Validate conventional PR title
         uses: amannn/action-semantic-pull-request@v5

.github/workflows/prebuild-devcontainers.yml

@@ -7,6 +7,8 @@ on:
       - '.github/workflows/prebuild-devcontainers.yml'
       - rust-toolchain.toml
       - Dockerfile
+    branches:
+      - master
   workflow_dispatch:
 
 concurrency:

.github/workflows/release-docker-image.yml

@@ -89,7 +89,7 @@ jobs:
   publish-manifest:
     name: Publish image tags
     needs: build-image
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Download digests
         uses: actions/download-artifact@v3

.github/workflows/release.yml

@@ -208,11 +208,11 @@ jobs:
       matrix:
         include:
           - package_type: tarballs
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: win
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: deb
-            os: ubuntu-22.04
+            os: ubuntu-24.04
           - package_type: macos
             os: macos-14
     steps:

.github/workflows/tests-build-image.yml

@@ -17,7 +17,8 @@ on:
 jobs:
   build-image:
     name: Build ${{ inputs.name }} image
-    runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
+#    runs-on: ["self-hosted", "linux", "arm64", "ubuntu-platform"]
+    runs-on: ubuntu-24.04
     steps:
       - name: Check out repo
         uses: actions/checkout@v4
@@ -41,7 +42,7 @@ jobs:
           image_name: ${{ inputs.image_name }}
           image_org: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com
           target: ${{ inputs.target }}
-          platform: linux/arm64
+          platform: linux/amd64
           push_tags: true
           dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
           dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/tests-codeql.yml

@@ -4,7 +4,7 @@ on:
 jobs:
   codeql:
     name: Run Code QL
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       actions: read
       contents: read

.github/workflows/tests-js-package.yml

@@ -17,7 +17,7 @@ on:
 jobs:
   lint:
     name: Linting
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read
@@ -51,7 +51,7 @@ jobs:
 
   test:
     name: Tests
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     permissions:
       id-token: write
       contents: read