feat: upload chainflip-engine shared libraries to S3 💾 (#5061) (#5067)

* feat: upload `chainflip-engine` shared libraries to S3 💾

* fix: wait for all build jobs and use correct artifact name 🐛

* chore: use dev bucket 👮

* fix: AWS IAM role ARN 🐛

* chore: set version dynamically

* fix: binary path

* chore: add dylib files to mainnet pre-release

* fix: use `bash` as shell

* fix: skip docker

* fix: be more precise about versions

* chore: use `lib` as S3 path

* fix: 🤦‍♂️

* fix: 🤦‍♂️

* fix: 🤦‍♂️

* chore: output raw version cos why not?

* fix: 🤦‍♂️

* fix: AAAAAAAH! 😭

* fix: AAAAAAAH! 😭

* chore: remove debugging statements 🚀

* chore: output raw version cos why not?

* chore: remove test step 🧹

* fix: dev apt publish version

* Update .github/workflows/_61_upload_engine_shared_libs.yml



* Update ci/scripts/extract_version.sh



* chore: address PR review comment 👮‍♂️

* chore: address PR review comment

* fix: s3 path 🐛

---------

Co-authored-by: kylezs <kyle@chainflip.io>

.github/workflows/_20_build.yml

@@ -57,7 +57,6 @@ jobs:
             ./target/${{ inputs.binary-subdir }}/libchainflip_engine_v*.so
             ./target/${{ inputs.binary-subdir }}/engine-runner
 
-
       - name: Upload runtime artifacts
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
         with:

.github/workflows/_60_upload.yml

@@ -26,12 +26,23 @@ jobs:
           name: chainflip-backend-bin
           path: assets/bin
 
+      - name: Download Mac M2 binaries 📥
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          name: chainflip-backend-bin-mac-m2
+          path: /tmp/assets/m2
+
       - name: Remove non production binaries 🗑
         run: |
           rm -f assets/bin/chainflip-engine-databases
           rm -f assets/bin/chainflip-ingress-egress-tracker
           rm -f assets/bin/generate-genesis-keys
 
+      - name: Move .dylib files 📂
+        run: |
+          mkdir -p assets/m2
+          mv /tmp/assets/m2/libchainflip_engine_v*.dylib assets/m2/
+
       - name: Download packages 📥
         uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
         with:

.github/workflows/_61_upload_engine_shared_libs.yml

@@ -0,0 +1,49 @@
+on:
+  workflow_call:
+
+permissions:
+  id-token: write
+  contents: read
+jobs:
+  upload:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout 🛒
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+
+      - name: Download Artifacts 📥
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          name: chainflip-backend-bin
+          path: chainflip-backend-bin
+
+      - name: Download M2 binaries 📥
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a
+        with:
+          name: chainflip-backend-bin-mac-m2
+          path: chainflip-backend-bin-mac-m2
+
+      - name: Set Version 🔦
+        id: version
+        shell: bash
+        run: |
+          chmod +x chainflip-backend-bin/chainflip-node
+          VERSION=$(./ci/scripts/extract_version.sh "$(./chainflip-backend-bin/chainflip-node --version)" full)
+          echo "Version: $VERSION"
+          echo "url_friendly_version=$(echo $VERSION | tr . _)" >> $GITHUB_OUTPUT
+
+      - name: Move shared library files 📂
+        run: |
+          mkdir -p /tmp/chainflip-engine-shared-libs/
+          mv chainflip-backend-bin/libchainflip_engine_v${{ steps.version.outputs.url_friendly_version }}.so /tmp/chainflip-engine-shared-libs/
+          mv chainflip-backend-bin-mac-m2/libchainflip_engine_v${{ steps.version.outputs.url_friendly_version }}.dylib /tmp/chainflip-engine-shared-libs/
+
+      - name: Configure AWS credentials using OIDC 🪪
+        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
+        with:
+          aws-region: eu-central-1
+          role-to-assume: arn:aws:iam::962042992619:role/chainflip-github-bot
+
+      - name: Upload engine shared libraries S3 🚀
+        run: |
+          aws s3 cp /tmp/chainflip-engine-shared-libs/ s3://repo.chainflip.io/lib/${{ steps.version.outputs.url_friendly_version }}/ --recursive

.github/workflows/ci-development.yml

@@ -31,6 +31,11 @@ jobs:
     secrets: inherit
     with:
       profile: "release"
+  build-m2:
+    uses: ./.github/workflows/_21_build_m2.yml
+    secrets: inherit
+    with:
+      profile: "release"
   # Used to test upgrades to this version from the latest release
   build-try-runtime:
     uses: ./.github/workflows/_20_build.yml

.github/workflows/release-perseverance.yml

@@ -61,3 +61,7 @@ jobs:
     uses: ./.github/workflows/_70_post_release.yml
     with:
       network: "perseverance"
+  upload-engine-shared-libs:
+    needs: [build, build-m2]
+    uses: ./.github/workflows/_61_upload_engine_shared_libs.yml
+    secrets: inherit

.github/workflows/release-sisyphos.yml

@@ -102,3 +102,7 @@ jobs:
     uses: ./.github/workflows/_70_post_release.yml
     with:
       network: "sisyphos"
+  upload-engine-shared-libs:
+    needs: [build, build-m2]
+    uses: ./.github/workflows/_61_upload_engine_shared_libs.yml
+    secrets: inherit

ci/scripts/extract_version.sh

@@ -2,6 +2,9 @@
 
 # Input string
 input_string="$1"
+# Version type to extract. default is major_minor
+
+version_type="${2:-major_minor}"
 
 # Regular expression to match version pattern (X.Y.Z or X.Y)
 # This pattern looks for numbers separated by dots.
@@ -15,6 +18,10 @@ fi
 
 # Extract major and minor version
 major_minor=$(echo "$version" | cut -d. -f1,2)
+full_version=$(echo "$version" | cut -d. -f1,2,3)
 
-# Output the major and minor version
-echo "$major_minor"
+if [[ "$version_type" == "major_minor" ]]; then
+    echo $major_minor
+else
+    echo $full_version
+fi