feat: PRO-474 broadcast safe mode (#3902)

Co-authored-by: dandanlen <3168260+dandanlen@users.noreply.github.com>

state-chain/cf-integration-tests/src/authorities.rs

@@ -268,6 +268,7 @@ fn authority_rotation_cannot_be_aborted_after_key_handover_but_stalls_on_safe_mo
 				                                                             * witnessing */
 				reputation: pallet_cf_reputation::PalletSafeMode::CODE_RED,
 				vault: pallet_cf_vaults::PalletSafeMode::CODE_RED,
+				broadcast: pallet_cf_broadcast::PalletSafeMode::CODE_RED,
 			};
 
 			assert_ok!(Environment::update_safe_mode(

state-chain/pallets/cf-broadcast/src/lib.rs

@@ -8,8 +8,11 @@ mod tests;
 
 pub mod weights;
 use cf_primitives::{BroadcastId, ThresholdSignatureRequestId};
+use cf_traits::impl_pallet_safe_mode;
 pub use weights::WeightInfo;
 
+impl_pallet_safe_mode!(PalletSafeMode; retry_enabled);
+
 use cf_chains::{ApiCall, Chain, ChainCrypto, FeeRefundCalculator, TransactionBuilder};
 use cf_traits::{
 	offence_reporting::OffenceReporter, Broadcaster, Chainflip, EpochInfo, EpochKey,
@@ -183,6 +186,12 @@ pub mod pallet {
 		/// Something that provides the current key for signing.
 		type KeyProvider: KeyProvider<<Self::TargetChain as Chain>::ChainCrypto>;
 
+		/// Safe Mode access.
+		type SafeMode: Get<PalletSafeMode>;
+
+		/// The save mode block margin
+		type SafeModeBlockMargin: Get<BlockNumberFor<Self>>;
+
 		/// The weights for the pallet
 		type WeightInfo: WeightInfo;
 	}
@@ -314,40 +323,50 @@ pub mod pallet {
 			// eventually. For outlying, unknown unknowns, these can be something governance can
 			// handle if absolutely necessary (though it likely never will be).
 			let expiries = Timeouts::<T, I>::take(block_number);
-			for attempt_id in expiries.iter() {
-				if let Some(attempt) = Self::take_awaiting_broadcast(*attempt_id) {
-					Self::deposit_event(Event::<T, I>::BroadcastAttemptTimeout {
-						broadcast_attempt_id: *attempt_id,
-					});
-					Self::start_next_broadcast_attempt(attempt);
+			if T::SafeMode::get().retry_enabled {
+				for attempt_id in expiries.iter() {
+					if let Some(attempt) = Self::take_awaiting_broadcast(*attempt_id) {
+						Self::deposit_event(Event::<T, I>::BroadcastAttemptTimeout {
+							broadcast_attempt_id: *attempt_id,
+						});
+						Self::start_next_broadcast_attempt(attempt);
+					}
 				}
+			} else {
+				Timeouts::<T, I>::insert(
+					block_number.saturating_add(T::SafeModeBlockMargin::get()),
+					expiries.clone(),
+				);
 			}
-
 			T::WeightInfo::on_initialize(expiries.len() as u32)
 		}
 
 		// We want to retry broadcasts when we have free block space.
 		fn on_idle(_block_number: BlockNumberFor<T>, remaining_weight: Weight) -> Weight {
-			let next_broadcast_weight = T::WeightInfo::start_next_broadcast_attempt();
+			if T::SafeMode::get().retry_enabled {
+				let next_broadcast_weight = T::WeightInfo::start_next_broadcast_attempt();
 
-			let num_retries_that_fit = remaining_weight
-				.ref_time()
-				.checked_div(next_broadcast_weight.ref_time())
-				.expect("start_next_broadcast_attempt weight should not be 0")
-				as usize;
+				let num_retries_that_fit = remaining_weight
+					.ref_time()
+					.checked_div(next_broadcast_weight.ref_time())
+					.expect("start_next_broadcast_attempt weight should not be 0")
+					as usize;
 
-			let mut retries = BroadcastRetryQueue::<T, I>::take();
+				let mut retries = BroadcastRetryQueue::<T, I>::take();
 
-			if retries.len() >= num_retries_that_fit {
-				BroadcastRetryQueue::<T, I>::put(retries.split_off(num_retries_that_fit));
-			}
+				if retries.len() >= num_retries_that_fit {
+					BroadcastRetryQueue::<T, I>::put(retries.split_off(num_retries_that_fit));
+				}
 
-			let retries_len = retries.len();
+				let retries_len = retries.len();
 
-			for retry in retries {
-				Self::start_next_broadcast_attempt(retry);
+				for retry in retries {
+					Self::start_next_broadcast_attempt(retry);
+				}
+				next_broadcast_weight.saturating_mul(retries_len as u64) as Weight
+			} else {
+				Weight::zero()
 			}
-			next_broadcast_weight.saturating_mul(retries_len as u64) as Weight
 		}
 	}
 

state-chain/pallets/cf-broadcast/src/mock.rs

@@ -2,7 +2,7 @@
 
 use std::cell::RefCell;
 
-use crate::{self as pallet_cf_broadcast, Instance1, PalletOffence};
+use crate::{self as pallet_cf_broadcast, Instance1, PalletOffence, PalletSafeMode};
 use cf_chains::{
 	eth::Ethereum,
 	evm::EvmCrypto,
@@ -12,15 +12,15 @@ use cf_chains::{
 	Chain, ChainCrypto,
 };
 use cf_traits::{
-	impl_mock_chainflip,
+	impl_mock_chainflip, impl_mock_runtime_safe_mode,
 	mocks::{signer_nomination::MockNominator, threshold_signer::MockThresholdSigner},
 	AccountRoleRegistry, EpochKey, KeyState, OnBroadcastReady,
 };
 use codec::{Decode, Encode};
 use frame_support::{parameter_types, traits::UnfilteredDispatchable};
 use frame_system::pallet_prelude::BlockNumberFor;
 use scale_info::TypeInfo;
-use sp_core::H256;
+use sp_core::{ConstU64, H256};
 use sp_runtime::traits::{BlakeTwo256, IdentityLookup};
 type Block = frame_system::mocking::MockBlock<Test>;
 
@@ -131,6 +131,8 @@ impl OnBroadcastReady<MockEthereum> for MockBroadcastReadyProvider {
 	type ApiCall = MockApiCall<MockEthereumChainCrypto>;
 }
 
+impl_mock_runtime_safe_mode! { broadcast: PalletSafeMode }
+
 impl pallet_cf_broadcast::Config<Instance1> for Test {
 	type RuntimeEvent = RuntimeEvent;
 	type RuntimeCall = RuntimeCall;
@@ -147,7 +149,9 @@ impl pallet_cf_broadcast::Config<Instance1> for Test {
 	type KeyProvider = MockKeyProvider;
 	type RuntimeOrigin = RuntimeOrigin;
 	type BroadcastCallable = MockCallback;
+	type SafeMode = MockRuntimeSafeMode;
 	type BroadcastReadyProvider = MockBroadcastReadyProvider;
+	type SafeModeBlockMargin = ConstU64<10>;
 }
 
 impl_mock_chainflip!(Test);

state-chain/pallets/cf-broadcast/src/tests.rs

@@ -3,7 +3,7 @@
 use crate::{
 	mock::*, AwaitingBroadcast, BroadcastAttemptCount, BroadcastAttemptId, BroadcastId,
 	BroadcastRetryQueue, Error, Event as BroadcastEvent, FailedBroadcasters, Instance1,
-	PalletOffence, RequestCallbacks, ThresholdSignatureData, TransactionFeeDeficit,
+	PalletOffence, RequestCallbacks, ThresholdSignatureData, Timeouts, TransactionFeeDeficit,
 	TransactionOutIdToBroadcastId, WeightInfo,
 };
 use cf_chains::{
@@ -16,7 +16,7 @@ use cf_chains::{
 };
 use cf_traits::{
 	mocks::{signer_nomination::MockNominator, threshold_signer::MockThresholdSigner},
-	AsyncResult, Chainflip, EpochInfo, ThresholdSigner,
+	AsyncResult, Chainflip, EpochInfo, SetSafeMode, ThresholdSigner,
 };
 use frame_support::{assert_noop, assert_ok, dispatch::Weight, traits::Hooks};
 use frame_system::RawOrigin;
@@ -513,3 +513,29 @@ fn threshold_sign_and_broadcast_with_callback() {
 		);
 	});
 }
+
+#[test]
+fn ensure_safe_mode_is_moving_timeouts() {
+	new_test_ext().execute_with(|| {
+		<MockRuntimeSafeMode as SetSafeMode<MockRuntimeSafeMode>>::set_code_red();
+		let _ = start_mock_broadcast();
+		assert!(Timeouts::<Test, Instance1>::get(5u64).len() == 1);
+		Broadcaster::on_initialize(5);
+		assert!(Timeouts::<Test, Instance1>::get(5u64).is_empty());
+		assert!(Timeouts::<Test, Instance1>::get(15u64).len() == 1);
+	});
+}
+
+#[test]
+fn ensure_retries_are_skipped_during_safe_mode() {
+	new_test_ext().execute_with(|| {
+		let _ = start_mock_broadcast();
+		MockCfe::respond(Scenario::SigningFailure);
+		let _ = start_mock_broadcast();
+		MockCfe::respond(Scenario::SigningFailure);
+		assert_eq!(BroadcastRetryQueue::<Test, Instance1>::get().len(), 2);
+		<MockRuntimeSafeMode as SetSafeMode<MockRuntimeSafeMode>>::set_code_red();
+		Broadcaster::on_idle(0, LARGE_EXCESS_WEIGHT);
+		assert_eq!(BroadcastRetryQueue::<Test, Instance1>::get().len(), 2);
+	});
+}

state-chain/runtime/src/chainflip.rs

@@ -86,6 +86,7 @@ impl_runtime_safe_mode! {
 	reputation: pallet_cf_reputation::PalletSafeMode,
 	vault: pallet_cf_vaults::PalletSafeMode,
 	witnesser: pallet_cf_witnesser::PalletSafeMode,
+	broadcast: pallet_cf_broadcast::PalletSafeMode,
 }
 struct BackupNodeEmissions;
 

state-chain/runtime/src/lib.rs

@@ -668,6 +668,8 @@ impl pallet_cf_broadcast::Config<EthereumInstance> for Runtime {
 	type BroadcastReadyProvider = BroadcastReadyProvider;
 	type BroadcastTimeout = ConstU32<{ 10 * MINUTES }>;
 	type WeightInfo = pallet_cf_broadcast::weights::PalletWeight<Runtime>;
+	type SafeMode = chainflip::RuntimeSafeMode;
+	type SafeModeBlockMargin = ConstU32<10>;
 	type KeyProvider = EthereumVault;
 }
 
@@ -688,6 +690,8 @@ impl pallet_cf_broadcast::Config<PolkadotInstance> for Runtime {
 	type BroadcastReadyProvider = BroadcastReadyProvider;
 	type BroadcastTimeout = ConstU32<{ 10 * MINUTES }>;
 	type WeightInfo = pallet_cf_broadcast::weights::PalletWeight<Runtime>;
+	type SafeMode = chainflip::RuntimeSafeMode;
+	type SafeModeBlockMargin = ConstU32<10>;
 	type KeyProvider = PolkadotVault;
 }
 
@@ -708,6 +712,8 @@ impl pallet_cf_broadcast::Config<BitcoinInstance> for Runtime {
 	type BroadcastReadyProvider = BroadcastReadyProvider;
 	type BroadcastTimeout = ConstU32<{ 90 * MINUTES }>;
 	type WeightInfo = pallet_cf_broadcast::weights::PalletWeight<Runtime>;
+	type SafeMode = chainflip::RuntimeSafeMode;
+	type SafeModeBlockMargin = ConstU32<10>;
 	type KeyProvider = BitcoinVault;
 }