Merge remote-tracking branch 'origin/main' into feature/limit-select-…

…utxo

* origin/main:
  feat(custom-rpc): add broker info [WEB-925] (#4560)
  chore: upgrade solana version (#4567)
  fix: continuous adapter (PRO-684) (#4503)
  fix: Wait for ThresholdSignature success before switching to NewKeysActivated (#4534)
  chore: remove un-needed serde derives from EncodedAddress (#4565)
  fix: more lenient max deposit fee in bouncer test (#4564)
  chore: build persa bins instead of fetch (#4554)
  feat: deploy L2 contracts upon localnet startup and send L2 TXs 📑 (#4558)
  feat: debug logs on runtime upgrade test (#4556)

.github/workflows/_40_post_check.yml

@@ -13,6 +13,7 @@ on:
 
 env:
   FORCE_COLOR: 1
+  SOLANA_VERSION: v1.17.22
 
 permissions:
   packages: read
@@ -67,7 +68,7 @@ jobs:
 
       - name: Install solana
         run: |
-          sh -c "$(curl -sSfL https://release.solana.com/v1.17.16/install)"
+          sh -c "$(curl -sSfL https://release.solana.com/$SOLANA_VERSION/install)"
 
       - name: Start a localnet 🚀
         env:

.github/workflows/release-perseverance.yml

@@ -10,20 +10,23 @@ concurrency:
 jobs:
   release-checks:
     uses: ./.github/workflows/_03_release_checks.yml
-  bins:
+  build:
     needs: [release-checks]
-    uses: ./.github/workflows/_02_retrieve-bins.yml
+    uses: ./.github/workflows/_20_build.yml
     secrets: inherit
+    with:
+      profile: "production"
+      binary-subdir: "production"
   docker:
-    needs: [bins]
+    needs: [build]
     uses: ./.github/workflows/_24_docker.yml
     with:
       network: "perseverance"
       environment: "production"
       publish_public_images: true
     secrets: inherit
   package:
-    needs: [bins]
+    needs: [build]
     uses: ./.github/workflows/_25_package.yml
     with:
       network: "perseverance"

bouncer/commands/check_witnesses.ts

@@ -16,6 +16,7 @@
 import { blake2AsHex } from '@polkadot/util-crypto';
 import { runWithTimeout, sleep, getChainflipApi } from '../shared/utils';
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 const witnessHash = new Set<any>();
 function hashCall(extrinsic: SubmittableExtrinsic<'promise', ISubmittableResult>) {
   const blakeHash = blake2AsHex(extrinsic.method.toU8a(), 256);
@@ -48,6 +49,7 @@ async function main(): Promise<void> {
       console.log(currentBlockNumber);
     }
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     signedBlock.block.extrinsics.forEach((ex: any, _index: any) => {
       if (ex.toHuman().method.method === 'witnessAtEpoch') {
         const callData = ex.toHuman().method.args.call;

bouncer/shared/try_runtime_upgrade.ts

@@ -14,6 +14,7 @@ function tryRuntimeCommand(runtimePath: string, blockParam: string, networkUrl:
   try {
     execSync(
       `try-runtime --runtime ${runtimePath} on-runtime-upgrade --disable-spec-version-check --disable-idempotency-checks --checks all ${blockParam} --uri ${networkUrl} 2> ${stderrFile}`,
+      { env: { ...process.env, RUST_LOG: 'runtime::executive=debug' } },
     );
     console.log(`try-runtime success for blockParam ${blockParam}`);
   } catch (e) {

bouncer/tests/broker_fee_collection_test.ts

@@ -32,9 +32,9 @@ const commissionBps = 1000; // 10%
 // Maximum expected deposit and withdrawal fees.
 // Values obtained from running this test on 1 node localnet.
 const maxDepositFee = {
-  [Assets.ETH]: BigInt(350000),
+  [Assets.ETH]: BigInt(3500000),
   [Assets.DOT]: BigInt(197300000),
-  [Assets.FLIP]: BigInt(300000000),
+  [Assets.FLIP]: BigInt('20000000000000000'),
   [Assets.BTC]: BigInt(190),
   [Assets.USDC]: BigInt(0), // Fee is too low for localnet, it rounds to 0
 };

engine/src/witness/common/chunked_chain_source/chunked_by_vault/continuous.rs

@@ -87,6 +87,15 @@ where
 					stream::unfold(
 						(chain_stream.fuse(), chain_client.clone(), epoch, unprocessed_indices, inprogress_indices, processed_indices),
 						move |(mut chain_stream, chain_client, mut epoch, mut unprocessed_indices, mut inprogress_indices, mut processed_indices)| async move {
+							let is_epoch_complete = |processed_indices: &RleBitmap<Self::Index>, end: Self::Index| {
+								processed_indices.is_superset(&{
+									let mut bitmap = RleBitmap::new(true);
+									bitmap.set_range(..epoch.info.1, false);
+									bitmap.set_range(end.., false);
+									bitmap
+								})
+							};
+
 							loop_select!(
 								let header = chain_stream.next_or_pending() => {
 									let highest_processed = processed_indices.iter(true).last().map_or(epoch.info.1, |highest_processed| std::cmp::max(highest_processed, epoch.info.1));
@@ -113,12 +122,10 @@ where
 
 									break Some((header, (chain_stream, chain_client, epoch, unprocessed_indices, inprogress_indices, processed_indices)))
 								},
-								if epoch.historic_signal.get().is_some() && processed_indices.is_superset(&{
-									let mut bitmap = RleBitmap::new(true);
-									bitmap.set_range(..epoch.info.1, false);
-									bitmap.set_range(epoch.historic_signal.get().unwrap().1.., false);
-									bitmap
-								}) => break None,
+								// Allows the stream to exit while waiting for blocks, if the epoch becomes historic
+								if let true = epoch.historic_signal.clone().wait().map(|(_, historic_at, _)| is_epoch_complete(&processed_indices, historic_at)) => {
+									break None
+								} else disable then if is_epoch_complete(&processed_indices, epoch.historic_signal.get().unwrap().1) => break None,
 								let (_, header) = inprogress_indices.next_or_pending() => {
 									processed_indices.set(header.index, true);
 									let _result = self.store.store(epoch.index, &processed_indices);

localnet/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   init:
-    image: ghcr.io/chainflip-io/chainflip-eth-contracts/localnet-initial-state:v1.0.0-${NODE_COUNT}
+    image: ghcr.io/chainflip-io/chainflip-eth-contracts/localnet-initial-state:v1.1.1-${NODE_COUNT}
     container_name: init
     platform: linux/amd64
     volumes:
@@ -16,6 +16,15 @@ services:
       - "/tmp/solana/test-ledger:/test-ledger"
     command: /bin/sh -c "cp -R /solana/ledger/* /test-ledger"
 
+  arb-init:
+    image: ghcr.io/chainflip-io/chainflip-eth-contracts/arbitrum-gizmo-shop:v1.1.1-${NODE_COUNT}
+    restart: always
+    container_name: arb-init
+    pull_policy: always
+    platform: linux/amd64
+    environment:
+      ARB_ENDPOINT: "http://sequencer:8547"
+
   redis:
     image: redis:6.2.6
     ports:
@@ -24,7 +33,7 @@ services:
       - "/tmp/chainflip/data/redis-data:/data"
 
   geth:
-    image: ghcr.io/chainflip-io/chainflip-eth-contracts/geth:v1.0.0-${NODE_COUNT}
+    image: ghcr.io/chainflip-io/chainflip-eth-contracts/geth:v1.1.1-${NODE_COUNT}
     container_name: geth
     platform: linux/amd64
     ports:

localnet/manage.sh

@@ -102,7 +102,8 @@ build-localnet() {
 
   mkdir -p /tmp/chainflip/
   touch /tmp/chainflip/debug.log
-
+  echo "🪢 Pulling Docker Images"
+  docker compose -f localnet/docker-compose.yml -p "chainflip-localnet" pull >>$DEBUG_OUTPUT_DESTINATION 2>&1
   echo "🔮 Initializing Network"
   docker compose -f localnet/docker-compose.yml -p "chainflip-localnet" up $INITIAL_CONTAINERS -d $additional_docker_compose_up_args >>$DEBUG_OUTPUT_DESTINATION 2>&1
   echo "🦺 Updating init state files permissions ..."
@@ -137,6 +138,8 @@ build-localnet() {
 
   echo "🦑 Waiting for Arbitrum nodes to start"
   docker compose -f localnet/docker-compose.yml -p "chainflip-localnet" up $ARB_CONTAINERS -d $additional_docker_compose_up_args >>$DEBUG_OUTPUT_DESTINATION 2>&1
+  echo "🪄 Deploying L2 Contracts"
+  docker compose -f localnet/docker-compose.yml -p "chainflip-localnet" up arb-init -d $additional_docker_compose_up_args >>$DEBUG_OUTPUT_DESTINATION 2>&1
 
 
   INIT_RPC_PORT=9944

state-chain/cf-integration-tests/src/authorities.rs

@@ -113,6 +113,9 @@ fn authority_rotates_with_correct_sequence() {
 				RotationPhase::ActivatingKeys(..)
 			));
 
+			// Wait for an extra block to allow TSS to complete, we switch to RotationComplete once
+			// that's done
+			testnet.move_forward_blocks(1);
 			assert_eq!(
 				AllVaults::status(),
 				AsyncResult::Ready(KeyRotationStatusOuter::RotationComplete),

state-chain/cf-integration-tests/src/broadcasting.rs

@@ -42,7 +42,7 @@ fn bitcoin_broadcast_delay_works() {
 			)
 			.unwrap();
 
-			let broadcast_id =
+			let (broadcast_id, _) =
 				<BitcoinBroadcaster as Broadcaster<Bitcoin>>::threshold_sign_and_broadcast(
 					bitcoin_call,
 				);

state-chain/chains/src/address.rs

@@ -61,9 +61,7 @@ impl ForeignChainAddress {
 	}
 }
 
-#[derive(
-	Serialize, Deserialize, Clone, Debug, PartialEq, Eq, Encode, Decode, TypeInfo, PartialOrd, Ord,
-)]
+#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode, TypeInfo, PartialOrd, Ord)]
 pub enum EncodedAddress {
 	Eth([u8; 20]),
 	Dot([u8; 32]),

state-chain/custom-rpc/src/lib.rs

@@ -33,8 +33,8 @@ use state_chain_runtime::{
 	chainflip::{BlockUpdate, Offence},
 	constants::common::TX_FEE_MULTIPLIER,
 	runtime_apis::{
-		CustomRuntimeApi, DispatchErrorWithMessage, FailingWitnessValidators,
-		LiquidityProviderInfo, RuntimeApiAccountInfoV2,
+		BrokerInfo, CustomRuntimeApi, DispatchErrorWithMessage, FailingWitnessValidators,
+		LiquidityProviderInfo, ValidatorInfo,
 	},
 	NetworkFee,
 };
@@ -60,6 +60,7 @@ pub enum RpcAccountInfo {
 	},
 	Broker {
 		flip_balance: NumberOrHex,
+		earned_fees: any::AssetMap<NumberOrHex>,
 	},
 	LiquidityProvider {
 		balances: any::AssetMap<NumberOrHex>,
@@ -89,8 +90,17 @@ impl RpcAccountInfo {
 		Self::Unregistered { flip_balance: balance.into() }
 	}
 
-	fn broker(balance: u128) -> Self {
-		Self::Broker { flip_balance: balance.into() }
+	fn broker(balance: u128, broker_info: BrokerInfo) -> Self {
+		Self::Broker {
+			flip_balance: balance.into(),
+			earned_fees: cf_chains::assets::any::AssetMap::try_from_iter(
+				broker_info
+					.earned_fees
+					.iter()
+					.map(|(asset, balance)| (*asset, (*balance).into())),
+			)
+			.unwrap(),
+		}
 	}
 
 	fn lp(info: LiquidityProviderInfo, network: NetworkEnvironment, balance: u128) -> Self {
@@ -109,7 +119,7 @@ impl RpcAccountInfo {
 		}
 	}
 
-	fn validator(info: RuntimeApiAccountInfoV2) -> Self {
+	fn validator(info: ValidatorInfo) -> Self {
 		Self::Validator {
 			flip_balance: info.balance.into(),
 			bond: info.bond.into(),
@@ -672,12 +682,14 @@ where
 				.unwrap_or(AccountRole::Unregistered)
 			{
 				AccountRole::Unregistered => RpcAccountInfo::unregistered(balance),
-				AccountRole::Broker => RpcAccountInfo::broker(balance),
+				AccountRole::Broker => {
+					let info = api.cf_broker_info(hash, account_id).map_err(to_rpc_error)?;
+
+					RpcAccountInfo::broker(balance, info)
+				},
 				AccountRole::LiquidityProvider => {
-					let info = api
-						.cf_liquidity_provider_info(hash, account_id)
-						.map_err(to_rpc_error)?
-						.expect("role already validated");
+					let info =
+						api.cf_liquidity_provider_info(hash, account_id).map_err(to_rpc_error)?;
 
 					RpcAccountInfo::lp(
 						info,
@@ -686,7 +698,7 @@ where
 					)
 				},
 				AccountRole::Validator => {
-					let info = api.cf_account_info_v2(hash, &account_id).map_err(to_rpc_error)?;
+					let info = api.cf_validator_info(hash, &account_id).map_err(to_rpc_error)?;
 
 					RpcAccountInfo::validator(info)
 				},
@@ -702,7 +714,7 @@ where
 		let account_info = self
 			.client
 			.runtime_api()
-			.cf_account_info_v2(self.unwrap_or_best(at), &account_id)
+			.cf_validator_info(self.unwrap_or_best(at), &account_id)
 			.map_err(to_rpc_error)?;
 
 		Ok(RpcAccountInfoV2 {
@@ -1309,7 +1321,19 @@ mod test {
 
 	#[test]
 	fn test_broker_serialization() {
-		insta::assert_display_snapshot!(serde_json::to_value(RpcAccountInfo::broker(0)).unwrap());
+		insta::assert_display_snapshot!(serde_json::to_value(RpcAccountInfo::broker(
+			0,
+			BrokerInfo {
+				earned_fees: vec![
+					(Asset::Eth, 0),
+					(Asset::Btc, 0),
+					(Asset::Flip, 1000000000000000000),
+					(Asset::Usdc, 0),
+					(Asset::Dot, 0),
+				]
+			}
+		))
+		.unwrap());
 	}
 
 	#[test]
@@ -1353,7 +1377,7 @@ mod test {
 
 	#[test]
 	fn test_validator_serialization() {
-		let validator = RpcAccountInfo::validator(RuntimeApiAccountInfoV2 {
+		let validator = RpcAccountInfo::validator(ValidatorInfo {
 			balance: FLIPPERINOS_PER_FLIP,
 			bond: FLIPPERINOS_PER_FLIP,
 			last_heartbeat: 0,

state-chain/custom-rpc/src/snapshots/custom_rpc__test__broker_serialization.snap

@@ -1,5 +1,5 @@
 ---
 source: state-chain/custom-rpc/src/lib.rs
-expression: "serde_json::to_value(RpcAccountInfo::broker(0)).unwrap()"
+expression: "serde_json::to_value(RpcAccountInfo::broker(0,\n            BrokerInfo {\n                earned_fees: vec![(Asset::Eth, 0), (Asset::Btc, 0),\n                    (Asset::Flip, 1000000000000000000), (Asset::Usdc, 0),\n                    (Asset::Dot, 0),],\n            })).unwrap()"
 ---
-{"flip_balance":"0x0","role":"broker"}
+{"earned_fees":{"Bitcoin":{"BTC":"0x0"},"Ethereum":{"ETH":"0x0","FLIP":"0xde0b6b3a7640000","USDC":"0x0"},"Polkadot":{"DOT":"0x0"}},"flip_balance":"0x0","role":"broker"}

state-chain/pallets/cf-broadcast/src/benchmarking.rs

@@ -128,7 +128,7 @@ mod benchmarks {
 
 	#[benchmark]
 	fn start_next_broadcast_attempt() {
-		let broadcast_id = Pallet::<T, I>::threshold_sign_and_broadcast(
+		let (broadcast_id, _) = Pallet::<T, I>::threshold_sign_and_broadcast(
 			BenchmarkValue::benchmark_value(),
 			None,
 			|_| None,

state-chain/pallets/cf-broadcast/src/lib.rs

@@ -664,7 +664,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 		maybe_failed_callback_generator: impl FnOnce(
 			BroadcastId,
 		) -> Option<<T as Config<I>>::BroadcastCallable>,
-	) -> BroadcastId {
+	) -> (BroadcastId, ThresholdSignatureRequestId) {
 		let broadcast_id = Self::next_broadcast_id();
 
 		PendingBroadcasts::<T, I>::append(broadcast_id);
@@ -676,9 +676,7 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
 			RequestFailureCallbacks::<T, I>::insert(broadcast_id, callback);
 		}
 
-		let _threshold_signature_id = Self::threshold_sign(api_call, broadcast_id, true);
-
-		broadcast_id
+		(broadcast_id, Self::threshold_sign(api_call, broadcast_id, true))
 	}
 
 	/// Signs a API call, use `Call::on_signature_ready` as the callback, and returns the signature
@@ -922,7 +920,9 @@ impl<T: Config<I>, I: 'static> Broadcaster<T::TargetChain> for Pallet<T, I> {
 	type ApiCall = T::ApiCall;
 	type Callback = <T as Config<I>>::BroadcastCallable;
 
-	fn threshold_sign_and_broadcast(api_call: Self::ApiCall) -> BroadcastId {
+	fn threshold_sign_and_broadcast(
+		api_call: Self::ApiCall,
+	) -> (BroadcastId, ThresholdSignatureRequestId) {
 		Self::threshold_sign_and_broadcast(api_call, None, |_| None)
 	}
 
@@ -931,7 +931,7 @@ impl<T: Config<I>, I: 'static> Broadcaster<T::TargetChain> for Pallet<T, I> {
 		success_callback: Option<Self::Callback>,
 		failed_callback_generator: impl FnOnce(BroadcastId) -> Option<Self::Callback>,
 	) -> BroadcastId {
-		Self::threshold_sign_and_broadcast(api_call, success_callback, failed_callback_generator)
+		Self::threshold_sign_and_broadcast(api_call, success_callback, failed_callback_generator).0
 	}
 
 	fn threshold_sign(api_call: Self::ApiCall) -> (BroadcastId, ThresholdSignatureRequestId) {
@@ -949,8 +949,11 @@ impl<T: Config<I>, I: 'static> Broadcaster<T::TargetChain> for Pallet<T, I> {
 		Self::clean_up_broadcast_storage(broadcast_id);
 	}
 
-	fn threshold_sign_and_broadcast_rotation_tx(api_call: Self::ApiCall) -> BroadcastId {
-		let broadcast_id = <Self as Broadcaster<_>>::threshold_sign_and_broadcast(api_call);
+	fn threshold_sign_and_broadcast_rotation_tx(
+		api_call: Self::ApiCall,
+	) -> (BroadcastId, ThresholdSignatureRequestId) {
+		let (broadcast_id, request_id) =
+			<Self as Broadcaster<_>>::threshold_sign_and_broadcast(api_call);
 
 		if let Some(earliest_pending_broadcast_id) = PendingBroadcasts::<T, I>::get()
 			.first()
@@ -962,6 +965,7 @@ impl<T: Config<I>, I: 'static> Broadcaster<T::TargetChain> for Pallet<T, I> {
 					}
 				}
 		}
-		broadcast_id
+
+		(broadcast_id, request_id)
 	}
 }