feat: end to end network upgrade github action (#4274)

* feat: upgrade testing in CI

* fix: ignore idempotency checks

* fix: fund_redeem test erasing decimals

* chore: clean up

* chore: lint

* chore: minor clean up

.github/workflows/ci-development.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches:
       - main
-      - 'release/*'
+      - "release/*"
 concurrency:
   group: ${{ github.ref }}-release-development
   cancel-in-progress: true

.github/workflows/upgrade-test.yml

@@ -0,0 +1,186 @@
+name: Test upgrade from latest release to main
+
+on:
+  workflow_dispatch:
+
+env:
+  FORCE_COLOR: 1
+
+permissions:
+  packages: read
+  contents: read
+
+jobs:
+  upgrade_test:
+    runs-on: [digitalocean]
+    # conservatively 1.5 hours. 2 bouncer runs need to occur.
+    timeout-minutes: 90
+    steps:
+
+      - name: Checkout chainflip-backend
+        uses: actions/checkout@v3
+
+      - name: Login to Github Container Registry 🔑
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Configure NodeJS
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: "pnpm"
+          registry-url: "https://npm.pkg.github.com"
+          scope: "@chainflip-io"
+          cache-dependency-path: "bouncer/pnpm-lock.yaml"
+
+      - name: Set NPM registry
+        run: |
+          pnpm set @chainflip-io:registry=https://npm.pkg.github.com/
+          pnpm set //npm.pkg.github.com/:_authToken=${{ secrets.GITHUB_TOKEN }}
+
+      - name: Install system packages
+        run: |
+          sudo apt update
+          sudo apt install -y bc xxd
+
+      - name: Install wscat
+        run: |
+          npm install -g wscat
+
+      - name: Download try-runtime binary
+        uses: jaxxstorm/action-install-gh-release@v1.10.0
+        with:
+          repo: paritytech/try-runtime-cli
+          tag: v0.5.0
+          extension-matching: disable
+          rename-to: try-runtime
+          chmod: 0755
+
+      - name: Download latest release binaries
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: release-perseverance.yml
+          name: chainflip-backend-bin-ubuntu-22.04
+          github_token: ${{ secrets.CF_BACKEND_GITHUB_TOKEN }}
+          path: latest-release-bins
+
+      - name: Permissions for latest binaries
+        run: |
+          chmod +x ./latest-release-bins/chainflip-*
+
+      - name: Version of the latest release
+        run: |
+          set -x
+          RELEASE_VERSION=$(./latest-release-bins/chainflip-engine --version)
+          echo $RELEASE_VERSION
+          echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV
+
+      - name: Download latest main binaries
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: ci-main.yml
+          name: chainflip-backend-bin-try-runtime-ubuntu-22.04
+          path: main-bins
+
+      - name: Permissions for latest binaries
+        run: |
+          chmod +x ./main-bins/chainflip-*
+
+      - name: Download latest main runtime
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: ci-main.yml
+          name: chainflip-node-runtime-try-runtime
+          path: main-runtime
+
+      - name: Install node dependencies
+        working-directory: bouncer
+        run: pnpm install
+
+      - name: Start a localnet from current release
+        env:
+          BINARY_ROOT_PATH: ./latest-release-bins
+        run: |
+          set -x
+          mkdir -p /tmp/chainflip/bashful
+          mkdir -p /tmp/chainflip/doc
+          mkdir -p /tmp/chainflip/dopey
+          touch ./localnet/.setup_complete
+          ./localnet/manage.sh
+
+      - name: Run bouncer on latest release
+        id: pre-upgrade-bouncer
+        working-directory: bouncer
+        run: |
+          ./run.sh
+
+      # we need to be sure that when this fails, we catch the error, any panics etc. that occur
+      # TODO: Run swaps simultaneously to the upgrade - we could do that inside the `upgrade_network` command itself.
+      - name: Upgrade network
+        shell: bash
+        id: upgrade-network
+        working-directory: bouncer
+        run: |
+          ./commands/upgrade_network.ts prebuilt \
+            --runtime ./../main-runtime/state_chain_runtime.compact.compressed.wasm \
+            --bins ./../main-bins \
+            --localnet_init ./../localnet/init \
+            --oldVersion "${{ env.RELEASE_VERSION }}"
+
+      - name: Run bouncer after upgrade
+        id: post-upgrade-bouncer
+        working-directory: bouncer
+        run: |
+          ./run-no-setup.sh
+
+      - name: Print chainflip-engine logs
+        if: failure()
+        run: |
+          cat /tmp/chainflip/*/chainflip-engine.log
+
+      - name: Print chainflip-node logs
+        if: failure()
+        run: |
+          cat /tmp/chainflip/*/chainflip-node.log
+
+      - name: Print broker logs
+        if: failure()
+        run: |
+          cat /tmp/chainflip/chainflip-broker-api.log
+
+      - name: Print lp-api logs
+        if: failure()
+        run: |
+          cat /tmp/chainflip/chainflip-lp-api.log
+
+      - name: Upload Localnet Logs 💾
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v3
+        with:
+          name: localnet-logs
+          path: |
+            /tmp/chainflip/*/chainflip-*.log
+
+      - name: Clean Up docker containers 🧹
+        if: always()
+        continue-on-error: true
+        run: |
+          ls -alR /tmp/chainflip
+          docker compose -f localnet/docker-compose.yml -p "chainflip-localnet" logs
+          docker compose -f localnet/docker-compose.yml -p "chainflip-localnet" down --rmi all --volumes --remove-orphans
+
+      - name: Notify on failed upgrade test
+        if: failure() && github.ref_name == 'main' || cancelled() && github.ref_name == 'main'
+        env:
+          DISCORD_USERNAME: "Upgrade Test"
+          DISCORD_WEBHOOK: ${{ secrets.CF_DISCORD_ALERTS_CRITICAL }}
+        uses: Ilshidur/action-discord@0.3.2
+        with:
+          args: |
+            ❗️❗️❗️❗️ Sorry **${{ github.actor }}**, The Upgrade Test has not passed ❗️❗️❗️❗️
+            👾 Link to job: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+            #️⃣ Tagging: <@&939151218708709416>

bouncer/shared/fund_redeem.ts

@@ -29,7 +29,7 @@ async function redeemAndObserve(
   await redeemFlip(seed, redeemEthAddress, redeemAmount);
 
   const newBalance = await observeBalanceIncrease('FLIP', redeemEthAddress, initBalance);
-  const balanceIncrease = newBalance - parseInt(initBalance);
+  const balanceIncrease = newBalance - parseFloat(initBalance);
   console.log(
     `Redemption success! New balance: ${newBalance.toString()}, Increase: ${balanceIncrease}`,
   );
@@ -68,7 +68,13 @@ export async function testFundRedeem(providedSeed?: string) {
     redeemEthAddress as HexString,
     exactRedeemAmount,
   );
-  assert.strictEqual(redeemedExact, exactAmount, `Unexpected balance increase amount`);
+  console.log(`Expected balance increase amount: ${exactAmount}`);
+  assert.strictEqual(
+    redeemedExact.toFixed(5),
+    exactAmount.toFixed(5),
+    `Unexpected balance increase amount`,
+  );
+  console.log('Redeem exact amount success!');
 
   // Test redeeming the rest of the flip with a 'Max' redeem amount
   console.log(`Testing redeem all`);

bouncer/shared/submit_runtime_upgrade.ts

@@ -44,7 +44,14 @@ export async function submitRuntimeUpgradeWithRestrictions(
   // TODO: Check if there were any errors in the submission, like `UpgradeConditionsNotMet` and `NotEnoughAuthoritiesCfesAtTargetVersion`.
   // and exit with error.
 
-  await observeEvent('system:CodeUpdated', chainflip);
+  const event = await Promise.race([
+    observeEvent('system:CodeUpdated', chainflip),
+    observeEvent('governance:FailedExecution', chainflip),
+  ]);
+
+  if (event.name === 'governance:FailedExecution') {
+    throw Error(`Runtime upgrade failed with error: ${event.data}`);
+  }
 
   console.log('Runtime upgrade completed.');
 }

bouncer/shared/try_runtime_upgrade.ts

@@ -6,10 +6,16 @@ import { execSync } from 'child_process';
 import { compileBinaries } from './utils/compile_binaries';
 
 function tryRuntimeCommand(runtimePath: string, blockParam: string, networkUrl: string) {
-  execSync(
-    `try-runtime --runtime ${runtimePath} on-runtime-upgrade --disable-spec-version-check --checks all ${blockParam} --uri ${networkUrl}`,
-    { stdio: 'ignore' },
-  );
+  try {
+    execSync(
+      `try-runtime --runtime ${runtimePath} on-runtime-upgrade --disable-spec-version-check --disable-idempotency-checks --checks all ${blockParam} --uri ${networkUrl}`,
+      { stdio: 'ignore' },
+    );
+  } catch (error) {
+    console.log('Error:', error);
+    console.log('Standard Error Output:', error.stderr);
+    console.log('Standard Output:', error.stdout);
+  }
 }
 
 // 4 options:

bouncer/shared/upgrade_network.ts

@@ -67,7 +67,6 @@ async function incompatibleUpgradeNoBuild(
     )}" LOCALNET_INIT_DIR=${localnetInitPath} BINARY_ROOT_PATH=${binaryPath} ${localnetInitPath}/scripts/start-all-engines.sh`,
   );
 
-  // let the engines do what they gotta do
   await sleep(7000);
 
   console.log('Engines started');
@@ -77,6 +76,17 @@ async function incompatibleUpgradeNoBuild(
   console.log(
     'Check that the old engine has now shut down, and that the new engine is now running.',
   );
+
+  execSync(`kill $(lsof -t -i:10997)`);
+  execSync(`kill $(lsof -t -i:10589)`);
+  await sleep(6000);
+  console.log('Stopped old broker and lp-api. Starting the new ones.');
+
+  const KEYS_DIR = `${localnetInitPath}/keys`;
+  execSync(`KEYS_DIR=${KEYS_DIR} ${localnetInitPath}/scripts/start-broker-api.sh ${binaryPath}`);
+  execSync(`KEYS_DIR=${KEYS_DIR} ${localnetInitPath}/scripts/start-lp-api.sh ${binaryPath}`);
+  await sleep(6000);
+  console.log('Started new broker and lp-api.');
 }
 
 async function incompatibleUpgrade(
@@ -173,8 +183,14 @@ export async function upgradeNetworkPrebuilt(
 ) {
   const versionRegex = /\d+\.\d+\.\d+/;
 
-  const cfeBinaryVersion = execSync(`${binariesPath}/chainflip-engine --version`).toString();
+  console.log("Version we're upgrading from: " + oldVersion);
 
+  let cleanOldVersion = oldVersion;
+  if (!versionRegex.test(cleanOldVersion)) {
+    cleanOldVersion = oldVersion.match(versionRegex)[0];
+  }
+
+  const cfeBinaryVersion = execSync(`${binariesPath}/chainflip-engine --version`).toString();
   const cfeVersion = cfeBinaryVersion.match(versionRegex)[0];
   console.log("CFE version we're upgrading to: " + cfeVersion);
 
@@ -188,13 +204,13 @@ export async function upgradeNetworkPrebuilt(
     );
   }
 
-  if (compareSemVer(oldVersion, cfeVersion) === 'greater') {
+  if (compareSemVer(cleanOldVersion, cfeVersion) === 'greater') {
     throw new Error(
       "The version we're upgrading to is older than the version we're upgrading from. Ensure you selected the correct binaries.",
     );
   }
 
-  const isCompatible = isCompatibleWith(oldVersion, cfeVersion);
+  const isCompatible = isCompatibleWith(cleanOldVersion, cfeVersion);
 
   if (!isCompatible) {
     console.log('The versions are incompatible.');

localnet/init/scripts/start-all-engines.sh

@@ -1,3 +1,5 @@
+#!/bin/bash
+
 # Starts all the engines necessary for the network, or for an upgrade.
 
 # These need to match what's in the manage.py script.

localnet/init/scripts/start-broker-api.sh

@@ -4,4 +4,4 @@ binary_location=$1
 RUST_LOG=debug,jsonrpsee_types::params=trace $binary_location/chainflip-broker-api \
   --port=10997 \
   --state_chain.ws_endpoint=ws://localhost:9944 \
-  --state_chain.signing_key_file localnet/init/keys/BROKER_1 > /tmp/chainflip/chainflip-broker-api.log 2>&1 &
+  --state_chain.signing_key_file $KEYS_DIR/BROKER_1 > /tmp/chainflip/chainflip-broker-api.log 2>&1 &

localnet/init/scripts/start-lp-api.sh

@@ -4,4 +4,4 @@ binary_location=$1
 RUST_LOG=debug,jsonrpsee_types::params=trace $binary_location/chainflip-lp-api \
   --port=10589 \
   --state_chain.ws_endpoint=ws://localhost:9944 \
-  --state_chain.signing_key_file localnet/init/keys/LP_1 > /tmp/chainflip/chainflip-lp-api.log 2>&1 &
+  --state_chain.signing_key_file $KEYS_DIR/LP_1 > /tmp/chainflip/chainflip-lp-api.log 2>&1 &

localnet/manage.sh

@@ -150,11 +150,13 @@ build-localnet() {
 
   wait
 
+  KEYS_DIR=./$LOCALNET_INIT_DIR/keys
+
   echo "🕺 Starting Broker API ..."
-  ./$LOCALNET_INIT_DIR/scripts/start-broker-api.sh $BINARY_ROOT_PATH
+  KEYS_DIR=$KEYS_DIR ./$LOCALNET_INIT_DIR/scripts/start-broker-api.sh $BINARY_ROOT_PATH
 
   echo "🤑 Starting LP API ..."
-  ./$LOCALNET_INIT_DIR/scripts/start-lp-api.sh $BINARY_ROOT_PATH
+  KEYS_DIR=$KEYS_DIR ./$LOCALNET_INIT_DIR/scripts/start-lp-api.sh $BINARY_ROOT_PATH
 
 
   print_success