chore: run CI on nscloud runners 🚀 (#4505)

* chore: test out namespace builds * chore: use big machine * chore: replace all nodes with namespace * chore: fix lint * chore: fix lint * chore: try rust cache * chore: try rust cache * debug: try using standard runners * chore: test without cache * chore: use rust builder * kick-off CI * fix: bouncer * chore: standerize runner tags * chore: faster clippy 🚀 * fix: add runner tag to actionlint * fix: duplicated debian package name 🐛 --------- Co-authored-by: Assem <asem.hasna@gmail.com>
chainflip-io · Mar 21, 2024 · 0998d74 · 0998d74
1 parent 20c8c62
commit 0998d74
Show file tree

Hide file tree

Showing 20 changed files with 66 additions and 44 deletions.
diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml
@@ -1,6 +1,6 @@
 self-hosted-runner:
   # Labels of self-hosted runner in array of strings.
-  labels: [hetzner, digitalocean]
+  labels: [hetzner, digitalocean, nscloud-ubuntu-22.04-amd64-2x4, nscloud-ubuntu-22.04-amd64-2x4, nscloud-ubuntu-22.04-amd64-4x8, namespace-profile-rust-builder, nscloud-ubuntu-22.04-amd64-32x64, nscloud-ubuntu-22.04-amd64-8x16, nscloud-ubuntu-22.04-amd64-4x16]
 # Configuration variables in array of strings defined in your repository or
 # organization. `null` means disabling configuration variables check.
 # Empty array means no configuration variable is allowed.

diff --git a/.github/workflows/_01_pre_check.yml b/.github/workflows/_01_pre_check.yml
@@ -9,7 +9,7 @@ env:
 
 jobs:
   cargo-fmt:
-    runs-on: [hetzner]
+    runs-on: nscloud-ubuntu-22.04-amd64-2x4
     container:
       image: "ghcr.io/${{ github.repository }}/rust-base:nightly-2024-02-04-ubuntu-20.04-44"
       credentials:
@@ -23,7 +23,7 @@ jobs:
         run: cargo fmt --all -- --check
 
   audit:
-    runs-on: [hetzner]
+    runs-on: nscloud-ubuntu-22.04-amd64-4x16
     container:
       image: "ghcr.io/${{ github.repository }}/rust-base:nightly-2024-02-04-ubuntu-20.04-44"
       credentials:
@@ -36,7 +36,7 @@ jobs:
         run: cargo cf-audit
 
   cargo-clippy:
-    runs-on: [hetzner]
+    runs-on: namespace-profile-rust-builder
     container:
       image: "ghcr.io/${{ github.repository }}/rust-base:nightly-2024-02-04-ubuntu-20.04-44"
       credentials:
@@ -51,7 +51,7 @@ jobs:
         run: cargo cf-clippy-ci
 
   lint-bouncer:
-    runs-on: [hetzner]
+    runs-on: ubuntu-22.04
     container:
       image: "ghcr.io/${{ github.repository }}/rust-base:nightly-2024-02-04-ubuntu-20.04-44"
       credentials:
@@ -61,6 +61,9 @@ jobs:
       - name: Checkout chainflip-backend
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
 
+      - name: Install pnpm
+        run: npm install -g pnpm
+
       - name: Configure NodeJS
         uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65
         with:
@@ -88,7 +91,7 @@ jobs:
         run: pnpm eslint:check
 
   lint-ci-workflows:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -100,7 +103,7 @@ jobs:
         shell: bash
 
   lint-dockerfiles:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         dockerfile:
@@ -123,7 +126,13 @@ jobs:
       - name: Checkout 🏁
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
 
+      - name: Login to DockerHub 🔑
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d
+        with:
+          username: ${{ secrets.CF_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.CF_DOCKERHUB_TOKEN }}
+
       - name: Lint 🐳🔬
-        uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf
+        uses: hadolint/hadolint-action@v3.1.0
         with:
           dockerfile: ci/docker/${{ matrix.environment }}/${{ matrix.dockerfile }}.Dockerfile
diff --git a/.github/workflows/_02_retrieve-bins.yml b/.github/workflows/_02_retrieve-bins.yml
@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         ubuntu_version: [20.04, 22.04]
-    runs-on: ubuntu-${{ matrix.ubuntu_version }}
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/_03_release_checks.yml b/.github/workflows/_03_release_checks.yml
@@ -11,7 +11,7 @@ env:
 
 jobs:
   check-version:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         manifest:
@@ -51,7 +51,7 @@ jobs:
           fi
 
   check-changelog:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     if: inputs.network != 'sisyphos'
     steps:
       - name: Checkout 🏁
@@ -62,7 +62,7 @@ jobs:
         run: ./ci/scripts/check_changelog.sh ${{ github.ref_name }}
 
   check-runtime-spec-version:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout 🏁
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/_04_merge_commit_summary.yml b/.github/workflows/_04_merge_commit_summary.yml
@@ -6,7 +6,7 @@ env:
 
 jobs:
   build-deb:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/_05_force_version_bump.yml b/.github/workflows/_05_force_version_bump.yml
@@ -4,11 +4,14 @@ on:
 
 jobs:
   force-version-bump:
-    runs-on: [digitalocean]
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout backend
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
 
+      - name: Install pnpm
+        run: npm install -g pnpm
+
       - name: Configure NodeJS
         uses: actions/setup-node@8f152de45cc393bb48ce5d89d36b731f54556e65
         with:
@@ -20,10 +23,10 @@ jobs:
       - name: Download latest release binaries
         uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d
         with:
-              workflow: release-perseverance.yml
-              name: chainflip-backend-bin-ubuntu-22.04
-              github_token: ${{ secrets.CF_BACKEND_GITHUB_TOKEN }}
-              path: latest-release-bins
+          workflow: release-perseverance.yml
+          name: chainflip-backend-bin-ubuntu-22.04
+          github_token: ${{ secrets.CF_BACKEND_GITHUB_TOKEN }}
+          path: latest-release-bins
       - name: Permissions for latest binaries
         run: |
             chmod +x ./latest-release-bins/chainflip-*

diff --git a/.github/workflows/_10_test.yml b/.github/workflows/_10_test.yml
@@ -10,12 +10,18 @@ env:
 
 jobs:
   unit-tests:
-    runs-on: [hetzner]
+    runs-on: namespace-profile-rust-builder
     container:
       image: "ghcr.io/${{ github.repository }}/rust-base:nightly-2024-02-04-ubuntu-20.04-44"
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
+      env:
+        NSC_CACHE_PATH: ${{ env.NSC_CACHE_PATH }} # env.NSC_CACHE_PATH contains the path to Cache Volume directory, that is `/cache`.
+      volumes:
+        - /cache:/cache # Where the Cache Volume is mounted.
+      options: --cap-add=SYS_ADMIN # Required to by nscloud-cache-action to call `mount`.
+
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/_11_coverage.yml b/.github/workflows/_11_coverage.yml
@@ -15,7 +15,7 @@ env:
 
 jobs:
   unit-tests:
-    runs-on: [hetzner]
+    runs-on: namespace-profile-rust-builder
     container:
       image: "ghcr.io/${{ github.repository }}/rust-base:nightly-2024-02-04-ubuntu-20.04-44"
       credentials:
@@ -34,7 +34,7 @@ jobs:
           sccache --show-stats
 
   coverage:
-    runs-on: [hetzner]
+    runs-on: namespace-profile-rust-builder
     container:
       image: "ghcr.io/${{ github.repository }}/rust-base:nightly-2024-02-04-ubuntu-20.04-44"
       credentials:

diff --git a/.github/workflows/_20_build.yml b/.github/workflows/_20_build.yml
@@ -23,7 +23,7 @@ env:
 
 jobs:
   compile:
-    runs-on: [hetzner]
+    runs-on: namespace-profile-rust-builder
     strategy:
       fail-fast: false
       matrix:
@@ -33,6 +33,12 @@ jobs:
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
+      env:
+        NSC_CACHE_PATH: ${{ env.NSC_CACHE_PATH }} # env.NSC_CACHE_PATH contains the path to Cache Volume directory, that is `/cache`.
+      volumes:
+        - /cache:/cache # Where the Cache Volume is mounted.
+      options: --cap-add=SYS_ADMIN # Required to by nscloud-cache-action to call `mount`.
+
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/_24_docker.yml b/.github/workflows/_24_docker.yml
@@ -85,7 +85,7 @@ jobs:
           - chainflip-engine-databases
         docker-repo:
           - ghcr.io/${{ github.repository }}
-    runs-on: ubuntu-22.04
+    runs-on: nscloud-ubuntu-22.04-amd64-4x8
     steps:
       - name: Checkout 🏁
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
@@ -170,7 +170,7 @@ jobs:
           - chainflip-lp-api
         docker-repo:
           - chainfliplabs
-    runs-on: ubuntu-22.04
+    runs-on: nscloud-ubuntu-22.04-amd64-4x8
     steps:
       - name: Checkout 🏁
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/_25_package.yml b/.github/workflows/_25_package.yml
@@ -11,7 +11,7 @@ env:
 
 jobs:
   packages:
-    runs-on: [hetzner]
+    runs-on: nscloud-ubuntu-22.04-amd64-4x8
     strategy:
       fail-fast: false
       matrix:
@@ -50,8 +50,3 @@ jobs:
           name: chainflip-backend-packages-ubuntu-${{ matrix.ubuntu_version }}
           path: |
             target/debian/*.deb
-
-      - name: Check if building changes any files
-        run: |
-          git diff --exit-code
-          git diff --cached --exit-code
diff --git a/.github/workflows/_40_post_check.yml b/.github/workflows/_40_post_check.yml
@@ -21,7 +21,7 @@ permissions:
 
 jobs:
   bouncer:
-    runs-on: [digitalocean]
+    runs-on: digitalocean
     timeout-minutes: ${{ inputs.timeout-minutes }}
     steps:
       - name: Checkout chainflip-backend

diff --git a/.github/workflows/_50_release.yml b/.github/workflows/_50_release.yml
@@ -10,7 +10,7 @@ env:
 
 jobs:
   compress-binaries:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -39,7 +39,7 @@ jobs:
           path: ${{ env.RELEASE_DIR }}.tar.gz
 
   release:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs:
       - compress-binaries
     steps:

diff --git a/.github/workflows/_60_upload.yml b/.github/workflows/_60_upload.yml
@@ -15,7 +15,7 @@ permissions:
   contents: read
 jobs:
   prerelease:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/_70_post_release.yml b/.github/workflows/_70_post_release.yml
@@ -12,7 +12,7 @@ permissions:
 
 jobs:
   invalidate-apt-cache:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Configure AWS credentials using OIDC 🪪
         uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a

diff --git a/.github/workflows/_90_helmcharts_check.yml b/.github/workflows/_90_helmcharts_check.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         chart: [chainflip-broker-api, chainflip-node]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout 🏁
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11

diff --git a/.github/workflows/ci-benchmarks.yml b/.github/workflows/ci-benchmarks.yml
@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: [hetzner]
+    runs-on: nscloud-ubuntu-22.04-amd64-4x8
     strategy:
       matrix:
         ubuntu_version: [20.04]
@@ -37,7 +37,7 @@ jobs:
             ./target/${{ matrix.profile }}/chainflip-node
 
   benchmarks:
-    runs-on: [hetzner]
+    runs-on: nscloud-ubuntu-22.04-amd64-4x8
     needs: [build]
     timeout-minutes: 120
     strategy:

diff --git a/.github/workflows/ci-cherry-pick.yml b/.github/workflows/ci-cherry-pick.yml
@@ -13,7 +13,7 @@ run-name: Cherry pick `${{ github.event.pull_request.title }}` to release branch
 jobs:
   cherry-pick:
     if: ${{ (github.event.pull_request.merged == true) && (contains(github.event.pull_request.labels.*.name, 'non-breaking')) }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout branch
         uses: actions/checkout@v3
@@ -32,12 +32,12 @@ jobs:
           git fetch --all
           COMMIT_SHA=$(git rev-parse HEAD)
           echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_ENV
-          
+
           RELEASE_BRANCH="release/$(git branch -r | grep 'origin/release/[0-9]\+\.[0-9]' | sed 's/.*release\///' | sort -V | tail -n1)"
           echo "RELEASE_BRANCH=$RELEASE_BRANCH" >> $GITHUB_ENV
-          
+
           git checkout $RELEASE_BRANCH
-          
+
           git cherry-pick -x $COMMIT_SHA
 
       - name: Create Pull Request

diff --git a/.github/workflows/docker-build-rust-base.yml b/.github/workflows/docker-build-rust-base.yml
@@ -17,7 +17,7 @@ jobs:
     permissions:
       contents: read
       packages: write
-    runs-on: ubuntu-${{ matrix.ubuntu_version }}
+    runs-on: nscloud-ubuntu-${{ matrix.ubuntu_version }}-amd64-4x8
     strategy:
       fail-fast: false
       matrix:

diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml
@@ -40,7 +40,7 @@ permissions:
 
 jobs:
   upgrade_test:
-    runs-on: [digitalocean]
+    runs-on: digitalocean
     # conservatively 1.5 hours. 2 bouncer runs need to occur.
     timeout-minutes: 90
     steps:
@@ -69,6 +69,9 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Install pnpm
+        run: npm install -g pnpm
+
       - name: Configure NodeJS
         uses: actions/setup-node@v3
         with: