From f4738dc78aa9fca0ec0b032fb0836431bdf75ebc Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 26 Jun 2024 09:15:36 +0300 Subject: [PATCH 01/55] Collation fetching fairness --- .../src/validator_side/collation.rs | 125 ++++++++++++++++-- .../src/validator_side/mod.rs | 61 ++++----- 2 files changed, 140 insertions(+), 46 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 96ffe9f13db3..748254e832f7 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -27,7 +27,12 @@ //! ┌──────────────────────────────────────────┐ //! └─▶Advertised ─▶ Pending ─▶ Fetched ─▶ Validated -use std::{collections::VecDeque, future::Future, pin::Pin, task::Poll}; +use std::{ + collections::{BTreeMap, VecDeque}, + future::Future, + pin::Pin, + task::Poll, +}; use futures::{future::BoxFuture, FutureExt}; use polkadot_node_network_protocol::{ @@ -48,6 +53,8 @@ use tokio_util::sync::CancellationToken; use crate::{error::SecondingError, LOG_TARGET}; +use super::GroupAssignments; + /// Candidate supplied with a para head it's built on top of. #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] pub struct ProspectiveCandidate { @@ -216,7 +223,6 @@ impl CollationStatus { } /// Information about collations per relay parent. -#[derive(Default)] pub struct Collations { /// What is the current status in regards to a collation for this relay parent? pub status: CollationStatus, @@ -225,18 +231,44 @@ pub struct Collations { /// This is the currently last started fetch, which did not exceed `MAX_UNSHARED_DOWNLOAD_TIME` /// yet. pub fetching_from: Option<(CollatorId, Option)>, - /// Collation that were advertised to us, but we did not yet fetch. - pub waiting_queue: VecDeque<(PendingCollation, CollatorId)>, + /// Collation that were advertised to us, but we did not yet fetch. Grouped by `ParaId`. + waiting_queue: BTreeMap>, /// How many collations have been seconded. pub seconded_count: usize, + /// What collations were fetched so far for this relay parent. + fetched_per_para: BTreeMap, + // Claims per `ParaId` for the assigned core at the relay parent. This information is obtained + // from the claim queue. + claims_per_para: BTreeMap, } impl Collations { + pub(super) fn new(assignments: &Vec) -> Self { + let mut claims_per_para = BTreeMap::new(); + for para_id in assignments { + *claims_per_para.entry(*para_id).or_default() += 1; + } + + Self { + status: Default::default(), + fetching_from: None, + waiting_queue: Default::default(), + seconded_count: 0, + fetched_per_para: Default::default(), + claims_per_para, + } + } + /// Note a seconded collation for a given para. pub(super) fn note_seconded(&mut self) { self.seconded_count += 1 } + // Note a collation which has been successfully fetched. + pub(super) fn note_fetched(&mut self, para_id: ParaId) { + *self.fetched_per_para.entry(para_id).or_default() += 1 + } + /// Returns the next collation to fetch from the `waiting_queue`. /// /// This will reset the status back to `Waiting` using [`CollationStatus::back_to_waiting`]. @@ -247,6 +279,7 @@ impl Collations { &mut self, finished_one: &(CollatorId, Option), relay_parent_mode: ProspectiveParachainsMode, + assignments: &GroupAssignments, ) -> Option<(PendingCollation, CollatorId)> { // If finished one does not match waiting_collation, then we already dequeued another fetch // to replace it. @@ -269,21 +302,20 @@ impl Collations { match self.status { // We don't need to fetch any other collation when we already have seconded one. CollationStatus::Seconded => None, - CollationStatus::Waiting => - if self.is_seconded_limit_reached(relay_parent_mode) { - None - } else { - self.waiting_queue.pop_front() - }, + CollationStatus::Waiting => self.pick_a_collation_to_fetch(&assignments.current), CollationStatus::WaitingOnValidation | CollationStatus::Fetching => unreachable!("We have reset the status above!"), } } - /// Checks the limit of seconded candidates. - pub(super) fn is_seconded_limit_reached( + /// Checks if another collation can be accepted. There are two limits: + /// 1. The number of collations that can be seconded. + /// 2. The number of collations that can be fetched per parachain. This is based on the number + /// of entries in the claim queue. + pub(super) fn is_collations_limit_reached( &self, relay_parent_mode: ProspectiveParachainsMode, + para_id: ParaId, ) -> bool { let seconded_limit = if let ProspectiveParachainsMode::Enabled { max_candidate_depth, .. } = @@ -293,7 +325,74 @@ impl Collations { } else { 1 }; - self.seconded_count >= seconded_limit + + let respected_per_para_limit = + self.claims_per_para.get(¶_id).copied().unwrap_or_default() >= + self.fetched_per_para.get(¶_id).copied().unwrap_or_default(); + + self.seconded_count >= seconded_limit || !respected_per_para_limit + } + + /// Adds a new collation to the waiting queue for the relay parent. This function doesn't + /// perform any limits check. The caller (`enqueue_collation`) should assure that the collation + /// can be enqueued. + pub(super) fn add_to_waiting_queue(&mut self, collation: (PendingCollation, CollatorId)) { + self.waiting_queue.entry(collation.0.para_id).or_default().push_back(collation); + } + + /// Picks a collation to fetch from the waiting queue. + /// When fetching collations we need to ensure that each parachain has got a fair core time + /// share depending on its assignments in the claim queue. This means that the number of + /// collations fetched per parachain should ideally be equal to (but not bigger than) the number + /// of claims for the particular parachain in the claim queue. + /// + /// To achieve this each parachain with at an entry in the `waiting_queue` has got a score + /// calculated by dividing the number of fetched collations by the number of entries in the + /// claim queue. Lower the score means higher fetching priority. Note that if a parachain hasn't + /// got anything fetched at this relay parent it will have score 0 which means highest priority. + /// + /// If two parachains has got the same score the one which is earlier in the claim queue will be + /// picked. + fn pick_a_collation_to_fetch( + &mut self, + claims: &Vec, + ) -> Option<(PendingCollation, CollatorId)> { + // Find the parachain(s) with the lowest score. + let mut lowest_score = None; + for (para_id, collations) in &mut self.waiting_queue { + let para_score = self + .fetched_per_para + .get(para_id) + .copied() + .unwrap_or_default() + .saturating_div(self.claims_per_para.get(para_id).copied().unwrap_or_default()); + + match lowest_score { + Some((score, _)) if para_score < score => + lowest_score = Some((para_score, vec![(para_id, collations)])), + Some((_, ref mut paras)) => { + paras.push((para_id, collations)); + }, + None => lowest_score = Some((para_score, vec![(para_id, collations)])), + } + } + + if let Some((_, mut lowest_score)) = lowest_score { + for claim in claims { + if let Some((_, collations)) = lowest_score.iter_mut().find(|(id, _)| *id == claim) + { + match collations.pop_front() { + Some(collation) => return Some(collation), + None => { + unreachable!("Collation can't be empty!") + }, + } + } + } + unreachable!("All entries in waiting_queue should also be in claim queue") + } else { + None + } } } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index f5c9726f3f6a..53bb665a0d71 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -374,12 +374,9 @@ struct PerRelayParent { } impl PerRelayParent { - fn new(mode: ProspectiveParachainsMode) -> Self { - Self { - prospective_parachains_mode: mode, - assignment: GroupAssignments { current: vec![] }, - collations: Collations::default(), - } + fn new(mode: ProspectiveParachainsMode, assignments: GroupAssignments) -> Self { + let collations = Collations::new(&assignments.current); + Self { prospective_parachains_mode: mode, assignment: assignments, collations } } } @@ -467,12 +464,11 @@ fn is_relay_parent_in_implicit_view( async fn assign_incoming( sender: &mut Sender, - group_assignment: &mut GroupAssignments, current_assignments: &mut HashMap, keystore: &KeystorePtr, relay_parent: Hash, relay_parent_mode: ProspectiveParachainsMode, -) -> Result<()> +) -> Result where Sender: CollatorProtocolSenderTrait, { @@ -499,7 +495,7 @@ where rotation_info.core_for_group(group, cores.len()) } else { gum::trace!(target: LOG_TARGET, ?relay_parent, "Not a validator"); - return Ok(()) + return Ok(GroupAssignments { current: Vec::new() }) }; let paras_now = match fetch_claim_queue(sender, relay_parent).await.map_err(Error::Runtime)? { @@ -532,9 +528,7 @@ where } } - *group_assignment = GroupAssignments { current: paras_now.into_iter().collect() }; - - Ok(()) + Ok(GroupAssignments { current: paras_now.into_iter().collect::>() }) } fn remove_outgoing( @@ -1107,7 +1101,10 @@ where ) .map_err(AdvertisementError::Invalid)?; - if per_relay_parent.collations.is_seconded_limit_reached(relay_parent_mode) { + if per_relay_parent + .collations + .is_collations_limit_reached(relay_parent_mode, para_id) + { return Err(AdvertisementError::SecondedLimitReached) } @@ -1199,7 +1196,7 @@ where }); let collations = &mut per_relay_parent.collations; - if collations.is_seconded_limit_reached(relay_parent_mode) { + if collations.is_collations_limit_reached(relay_parent_mode, para_id) { gum::trace!( target: LOG_TARGET, peer_id = ?peer_id, @@ -1222,9 +1219,11 @@ where ?relay_parent, "Added collation to the pending list" ); - collations.waiting_queue.push_back((pending_collation, collator_id)); + collations.add_to_waiting_queue((pending_collation, collator_id)); }, CollationStatus::Waiting => { + // We were waiting for a collation to be advertised to us (we were idle) so we can fetch + // the new collation immediately fetch_collation(sender, state, pending_collation, collator_id).await?; }, CollationStatus::Seconded if relay_parent_mode.is_enabled() => { @@ -1270,19 +1269,11 @@ where state.span_per_relay_parent.insert(*leaf, per_leaf_span); } - let mut per_relay_parent = PerRelayParent::new(mode); - assign_incoming( - sender, - &mut per_relay_parent.assignment, - &mut state.current_assignments, - keystore, - *leaf, - mode, - ) - .await?; + let assignments = + assign_incoming(sender, &mut state.current_assignments, keystore, *leaf, mode).await?; state.active_leaves.insert(*leaf, mode); - state.per_relay_parent.insert(*leaf, per_relay_parent); + state.per_relay_parent.insert(*leaf, PerRelayParent::new(mode, assignments)); if mode.is_enabled() { state @@ -1298,10 +1289,8 @@ where .unwrap_or_default(); for block_hash in allowed_ancestry { if let Entry::Vacant(entry) = state.per_relay_parent.entry(*block_hash) { - let mut per_relay_parent = PerRelayParent::new(mode); - assign_incoming( + let assignments = assign_incoming( sender, - &mut per_relay_parent.assignment, &mut state.current_assignments, keystore, *block_hash, @@ -1309,7 +1298,7 @@ where ) .await?; - entry.insert(per_relay_parent); + entry.insert(PerRelayParent::new(mode, assignments)); } } } @@ -1665,6 +1654,10 @@ async fn run_inner( let CollationEvent {collator_id, pending_collation, .. } = res.collation_event.clone(); + state.per_relay_parent.get_mut(&pending_collation.relay_parent).map(|rp_state| { + rp_state.collations.note_fetched(pending_collation.para_id); + }); + match kick_off_seconding(&mut ctx, &mut state, res).await { Err(err) => { gum::warn!( @@ -1737,9 +1730,11 @@ async fn dequeue_next_collation_and_fetch( previous_fetch: (CollatorId, Option), ) { while let Some((next, id)) = state.per_relay_parent.get_mut(&relay_parent).and_then(|state| { - state - .collations - .get_next_collation_to_fetch(&previous_fetch, state.prospective_parachains_mode) + state.collations.get_next_collation_to_fetch( + &previous_fetch, + state.prospective_parachains_mode, + &state.assignment, + ) }) { gum::debug!( target: LOG_TARGET, From c7074daac3c0e1c3b3ebc1616a7941b50ad03955 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 26 Jun 2024 10:19:56 +0300 Subject: [PATCH 02/55] Comments --- .../node/network/collator-protocol/src/validator_side/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 53bb665a0d71..f260d31319ea 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -1227,8 +1227,8 @@ where fetch_collation(sender, state, pending_collation, collator_id).await?; }, CollationStatus::Seconded if relay_parent_mode.is_enabled() => { - // Limit is not reached, it's allowed to second another - // collation. + // Limit is not reached (checked with `is_collations_limit_reached` before the match + // expression), it's allowed to second another collation. fetch_collation(sender, state, pending_collation, collator_id).await?; }, CollationStatus::Seconded => { From 73eee8702c783afd28f5502881f09e506f041942 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 26 Jun 2024 16:39:15 +0300 Subject: [PATCH 03/55] Fix tests and add some logs --- .../src/validator_side/collation.rs | 27 +++++- .../src/validator_side/tests/mod.rs | 24 +++++- .../tests/prospective_parachains.rs | 86 +++++++++++++++---- 3 files changed, 116 insertions(+), 21 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 748254e832f7..bf7df343be27 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -330,7 +330,20 @@ impl Collations { self.claims_per_para.get(¶_id).copied().unwrap_or_default() >= self.fetched_per_para.get(¶_id).copied().unwrap_or_default(); - self.seconded_count >= seconded_limit || !respected_per_para_limit + let respected_seconding_limit = self.seconded_count < seconded_limit; + + gum::trace!( + target: LOG_TARGET, + ?para_id, + claims_per_para=?self.claims_per_para, + fetched_per_para=?self.fetched_per_para, + ?seconded_limit, + ?respected_per_para_limit, + ?respected_seconding_limit, + "is_collations_limit_reached" + ); + + !(respected_seconding_limit && respected_per_para_limit) } /// Adds a new collation to the waiting queue for the relay parent. This function doesn't @@ -357,6 +370,13 @@ impl Collations { &mut self, claims: &Vec, ) -> Option<(PendingCollation, CollatorId)> { + gum::trace!( + target: LOG_TARGET, + waiting_queue = ?self.waiting_queue, + claim_queue = ?claims, + "Pick a collation to fetch." + ); + // Find the parachain(s) with the lowest score. let mut lowest_score = None; for (para_id, collations) in &mut self.waiting_queue { @@ -367,6 +387,11 @@ impl Collations { .unwrap_or_default() .saturating_div(self.claims_per_para.get(para_id).copied().unwrap_or_default()); + // skip empty queues + if collations.is_empty() { + continue + } + match lowest_score { Some((score, _)) if para_score < score => lowest_score = Some((para_score, vec![(para_id, collations)])), diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 44e25efd4dfc..4b59c9a507be 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -42,8 +42,9 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt}; use polkadot_primitives::{ - CandidateReceipt, CollatorPair, CoreIndex, CoreState, GroupIndex, GroupRotationInfo, HeadData, - OccupiedCore, PersistedValidationData, ScheduledCore, ValidatorId, ValidatorIndex, + AsyncBackingParams, CandidateReceipt, CollatorPair, CoreIndex, CoreState, GroupIndex, + GroupRotationInfo, HeadData, OccupiedCore, PersistedValidationData, ScheduledCore, ValidatorId, + ValidatorIndex, }; use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt_bad_sig, dummy_hash, @@ -77,6 +78,7 @@ struct TestState { group_rotation_info: GroupRotationInfo, cores: Vec, claim_queue: BTreeMap>, + async_backing_params: AsyncBackingParams, } impl Default for TestState { @@ -126,10 +128,23 @@ impl Default for TestState { }), ]; + let async_backing_params = + AsyncBackingParams { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let mut claim_queue = BTreeMap::new(); - claim_queue.insert(CoreIndex(0), [chain_ids[0]].into_iter().collect()); + claim_queue.insert( + CoreIndex(0), + iter::repeat(chain_ids[0]) + .take(async_backing_params.max_candidate_depth as usize + 1) + .collect(), + ); claim_queue.insert(CoreIndex(1), VecDeque::new()); - claim_queue.insert(CoreIndex(2), [chain_ids[1]].into_iter().collect()); + claim_queue.insert( + CoreIndex(2), + iter::repeat(chain_ids[1]) + .take(async_backing_params.max_candidate_depth as usize + 1) + .collect(), + ); Self { chain_ids, @@ -140,6 +155,7 @@ impl Default for TestState { group_rotation_info, cores, claim_queue, + async_backing_params, } } } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 472731b506ab..6e716a82b015 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -25,9 +25,6 @@ use polkadot_primitives::{ }; use rstest::rstest; -const ASYNC_BACKING_PARAMETERS: AsyncBackingParams = - AsyncBackingParams { max_candidate_depth: 4, allowed_ancestry_len: 3 }; - fn get_parent_hash(hash: Hash) -> Hash { Hash::from_low_u64_be(hash.to_low_u64_be() + 1) } @@ -98,8 +95,9 @@ async fn assert_assign_incoming( pub(super) async fn update_view( virtual_overseer: &mut VirtualOverseer, test_state: &TestState, - new_view: Vec<(Hash, u32)>, // Hash and block number. - activated: u8, // How many new heads does this update contain? + new_view: Vec<(Hash, u32)>, // Hash and block number. + activated: u8, // How many new heads does this update contain? + async_backing_params: &AsyncBackingParams, // returned via the runtime api ) -> Option { let new_view: HashMap = HashMap::from_iter(new_view); @@ -120,7 +118,7 @@ pub(super) async fn update_view( parent, RuntimeApiRequest::AsyncBackingParams(tx), )) => { - tx.send(Ok(ASYNC_BACKING_PARAMETERS)).unwrap(); + tx.send(Ok(async_backing_params.clone())).unwrap(); (parent, new_view.get(&parent).copied().expect("Unknown parent requested")) } ); @@ -134,7 +132,7 @@ pub(super) async fn update_view( ) .await; - let min_number = leaf_number.saturating_sub(ASYNC_BACKING_PARAMETERS.allowed_ancestry_len); + let min_number = leaf_number.saturating_sub(async_backing_params.allowed_ancestry_len); let ancestry_len = leaf_number + 1 - min_number; let ancestry_hashes = std::iter::successors(Some(leaf_hash), |h| Some(get_parent_hash(*h))) @@ -339,7 +337,14 @@ fn v1_advertisement_accepted_and_seconded() { let head_b = Hash::from_low_u64_be(128); let head_b_num: u32 = 0; - update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); @@ -418,7 +423,14 @@ fn v1_advertisement_rejected_on_non_active_leave() { let head_b = Hash::from_low_u64_be(128); let head_b_num: u32 = 5; - update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); @@ -468,7 +480,14 @@ fn accept_advertisements_from_implicit_view() { let head_d = get_parent_hash(head_c); // Activated leaf is `b`, but the collation will be based on `c`. - update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); let peer_b = PeerId::random(); @@ -570,7 +589,14 @@ fn second_multiple_candidates_per_relay_parent() { let head_c = Hash::from_low_u64_be(130); // Activated leaf is `b`, but the collation will be based on `c`. - update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); @@ -583,7 +609,7 @@ fn second_multiple_candidates_per_relay_parent() { ) .await; - for i in 0..(ASYNC_BACKING_PARAMETERS.max_candidate_depth + 1) { + for i in 0..(test_state.async_backing_params.max_candidate_depth + 1) { let mut candidate = dummy_candidate_receipt_bad_sig(head_c, Some(Default::default())); candidate.descriptor.para_id = test_state.chain_ids[0]; candidate.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); @@ -727,7 +753,14 @@ fn fetched_collation_sanity_check() { let head_c = Hash::from_low_u64_be(130); // Activated leaf is `b`, but the collation will be based on `c`. - update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); @@ -831,7 +864,14 @@ fn sanity_check_invalid_parent_head_data() { let head_c = Hash::from_low_u64_be(130); let head_c_num = 3; - update_view(&mut virtual_overseer, &test_state, vec![(head_c, head_c_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_c, head_c_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); @@ -954,7 +994,14 @@ fn advertisement_spam_protection() { let head_c = get_parent_hash(head_b); // Activated leaf is `b`, but the collation will be based on `c`. - update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); connect_and_declare_collator( @@ -1032,7 +1079,14 @@ fn child_blocked_from_seconding_by_parent(#[case] valid_parent: bool) { let head_c = Hash::from_low_u64_be(130); // Activated leaf is `b`, but the collation will be based on `c`. - update_view(&mut virtual_overseer, &test_state, vec![(head_b, head_b_num)], 1).await; + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; let peer_a = PeerId::random(); From fa321cef58fb9eec9c5eead030c0a92459c5c5e1 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 27 Jun 2024 13:38:40 +0300 Subject: [PATCH 04/55] Fix per para limit calculation in `is_collations_limit_reached` --- .../src/validator_side/collation.rs | 23 ++++++++++++++++--- .../src/validator_side/mod.rs | 11 +++++---- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index bf7df343be27..cbb926939d99 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -28,7 +28,7 @@ //! └─▶Advertised ─▶ Pending ─▶ Fetched ─▶ Validated use std::{ - collections::{BTreeMap, VecDeque}, + collections::{BTreeMap, HashMap, HashSet, VecDeque}, future::Future, pin::Pin, task::Poll, @@ -53,7 +53,7 @@ use tokio_util::sync::CancellationToken; use crate::{error::SecondingError, LOG_TARGET}; -use super::GroupAssignments; +use super::{GroupAssignments, PeerData}; /// Candidate supplied with a para head it's built on top of. #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] @@ -316,6 +316,7 @@ impl Collations { &self, relay_parent_mode: ProspectiveParachainsMode, para_id: ParaId, + peer_data: &HashMap, ) -> bool { let seconded_limit = if let ProspectiveParachainsMode::Enabled { max_candidate_depth, .. } = @@ -326,9 +327,24 @@ impl Collations { 1 }; + // All collators in `Collating` state for `para_id` we know about + let collators_for_para = peer_data + .iter() + .filter(|(_, data)| data.collating_para() == Some(para_id)) + .filter_map(|(_, data)| data.collator_id()) + .collect::>(); + + // If there is a pending fetch - count it + let pending_fetch = self + .fetching_from + .as_ref() + .map(|(collator_id, _)| if collators_for_para.contains(&collator_id) { 1 } else { 0 }) + .unwrap_or(0); + + // Successful fetches + a pending fetch < claim queue entries for `para_id` let respected_per_para_limit = self.claims_per_para.get(¶_id).copied().unwrap_or_default() >= - self.fetched_per_para.get(¶_id).copied().unwrap_or_default(); + self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + pending_fetch; let respected_seconding_limit = self.seconded_count < seconded_limit; @@ -337,6 +353,7 @@ impl Collations { ?para_id, claims_per_para=?self.claims_per_para, fetched_per_para=?self.fetched_per_para, + ?pending_fetch, ?seconded_limit, ?respected_per_para_limit, ?respected_seconding_limit, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index f260d31319ea..823e21ff1377 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -1101,10 +1101,11 @@ where ) .map_err(AdvertisementError::Invalid)?; - if per_relay_parent - .collations - .is_collations_limit_reached(relay_parent_mode, para_id) - { + if per_relay_parent.collations.is_collations_limit_reached( + relay_parent_mode, + para_id, + &state.peer_data, + ) { return Err(AdvertisementError::SecondedLimitReached) } @@ -1196,7 +1197,7 @@ where }); let collations = &mut per_relay_parent.collations; - if collations.is_collations_limit_reached(relay_parent_mode, para_id) { + if collations.is_collations_limit_reached(relay_parent_mode, para_id, &state.peer_data) { gum::trace!( target: LOG_TARGET, peer_id = ?peer_id, From 96392a574b32f60e220f1650e7f696434f5e1090 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 27 Jun 2024 13:39:30 +0300 Subject: [PATCH 05/55] Fix default `TestState` initialization: claim queue len should be equal to `allowed_ancestry_len` --- .../network/collator-protocol/src/validator_side/tests/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 4b59c9a507be..1da035ebc837 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -135,14 +135,14 @@ impl Default for TestState { claim_queue.insert( CoreIndex(0), iter::repeat(chain_ids[0]) - .take(async_backing_params.max_candidate_depth as usize + 1) + .take(async_backing_params.allowed_ancestry_len as usize) .collect(), ); claim_queue.insert(CoreIndex(1), VecDeque::new()); claim_queue.insert( CoreIndex(2), iter::repeat(chain_ids[1]) - .take(async_backing_params.max_candidate_depth as usize + 1) + .take(async_backing_params.allowed_ancestry_len as usize) .collect(), ); From 0f28aa8426037e0cf3532607982cc213d0849529 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 27 Jun 2024 16:21:13 +0300 Subject: [PATCH 06/55] clippy --- .../src/validator_side/tests/prospective_parachains.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 6e716a82b015..f5101ca98c1a 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -118,7 +118,7 @@ pub(super) async fn update_view( parent, RuntimeApiRequest::AsyncBackingParams(tx), )) => { - tx.send(Ok(async_backing_params.clone())).unwrap(); + tx.send(Ok(*async_backing_params)).unwrap(); (parent, new_view.get(&parent).copied().expect("Unknown parent requested")) } ); From e5ea548e944d912812bc5f7b169549a9a84799a3 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Fri, 28 Jun 2024 12:25:27 +0300 Subject: [PATCH 07/55] Update `is_collations_limit_reached` - remove seconded limit --- .../src/validator_side/collation.rs | 27 +++++++------------ .../src/validator_side/mod.rs | 9 ++----- .../tests/prospective_parachains.rs | 15 +++-------- 3 files changed, 16 insertions(+), 35 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index cbb926939d99..ff27991fc758 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -308,24 +308,21 @@ impl Collations { } } - /// Checks if another collation can be accepted. There are two limits: - /// 1. The number of collations that can be seconded. - /// 2. The number of collations that can be fetched per parachain. This is based on the number - /// of entries in the claim queue. + /// Checks if another collation can be accepted. The number of collations that can be fetched + /// per parachain is limited by the entries in the claim queue for the `ParaId` in question. + /// + /// If prospective parachains mode is not enabled then we fall back to synchronous backing. In + /// this case there is a limit of 1 collation per relay parent. pub(super) fn is_collations_limit_reached( &self, relay_parent_mode: ProspectiveParachainsMode, para_id: ParaId, peer_data: &HashMap, ) -> bool { - let seconded_limit = - if let ProspectiveParachainsMode::Enabled { max_candidate_depth, .. } = - relay_parent_mode - { - max_candidate_depth + 1 - } else { - 1 - }; + if let ProspectiveParachainsMode::Disabled = relay_parent_mode { + // fallback to synchronous backing + return self.seconded_count >= 1 + } // All collators in `Collating` state for `para_id` we know about let collators_for_para = peer_data @@ -346,21 +343,17 @@ impl Collations { self.claims_per_para.get(¶_id).copied().unwrap_or_default() >= self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + pending_fetch; - let respected_seconding_limit = self.seconded_count < seconded_limit; - gum::trace!( target: LOG_TARGET, ?para_id, claims_per_para=?self.claims_per_para, fetched_per_para=?self.fetched_per_para, ?pending_fetch, - ?seconded_limit, ?respected_per_para_limit, - ?respected_seconding_limit, "is_collations_limit_reached" ); - !(respected_seconding_limit && respected_per_para_limit) + !respected_per_para_limit } /// Adds a new collation to the waiting queue for the relay parent. This function doesn't diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 823e21ff1377..8da0faefe1bd 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -146,6 +146,7 @@ enum InsertAdvertisementError { /// No prior declare message received. UndeclaredCollator, /// A limit for announcements per peer is reached. + #[allow(dead_code)] PeerLimitReached, } @@ -250,10 +251,7 @@ impl PeerData { .advertisements .insert(on_relay_parent, HashSet::from_iter(candidate_hash)); }, - ( - ProspectiveParachainsMode::Enabled { max_candidate_depth, .. }, - candidate_hash, - ) => { + (ProspectiveParachainsMode::Enabled { .. }, candidate_hash) => { if let Some(candidate_hash) = candidate_hash { if state .advertisements @@ -266,9 +264,6 @@ impl PeerData { let candidates = state.advertisements.entry(on_relay_parent).or_default(); - if candidates.len() > max_candidate_depth { - return Err(InsertAdvertisementError::PeerLimitReached) - } candidates.insert(candidate_hash); } else { if self.version != CollationVersion::V1 { diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index f5101ca98c1a..cd3b4ec8d054 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -609,7 +609,7 @@ fn second_multiple_candidates_per_relay_parent() { ) .await; - for i in 0..(test_state.async_backing_params.max_candidate_depth + 1) { + for i in 0..(test_state.async_backing_params.allowed_ancestry_len) { let mut candidate = dummy_candidate_receipt_bad_sig(head_c, Some(Default::default())); candidate.descriptor.para_id = test_state.chain_ids[0]; candidate.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); @@ -694,16 +694,9 @@ fn second_multiple_candidates_per_relay_parent() { ) .await; - // Reported because reached the limit of advertisements per relay parent. - assert_matches!( - overseer_recv(&mut virtual_overseer).await, - AllMessages::NetworkBridgeTx( - NetworkBridgeTxMessage::ReportPeer(ReportPeerMessage::Single(peer_id, rep)), - ) => { - assert_eq!(peer_a, peer_id); - assert_eq!(rep.value, COST_UNEXPECTED_MESSAGE.cost_or_benefit()); - } - ); + // Rejected but not reported because reached the limit of advertisements for the para_id + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); // By different peer too (not reported). let pair_b = CollatorPair::generate().0; From 9abc898194f4242b7e671b4e92f52a8f4f56869e Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 1 Jul 2024 13:59:37 +0300 Subject: [PATCH 08/55] Fix pending fetches and more tests --- .../src/validator_side/collation.rs | 37 +- .../src/validator_side/mod.rs | 22 +- .../src/validator_side/tests/collation.rs | 61 +++ .../src/validator_side/tests/mod.rs | 73 +++- .../tests/prospective_parachains.rs | 365 ++++++++++++++---- 5 files changed, 450 insertions(+), 108 deletions(-) create mode 100644 polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index ff27991fc758..629a4eeb1305 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -28,7 +28,7 @@ //! └─▶Advertised ─▶ Pending ─▶ Fetched ─▶ Validated use std::{ - collections::{BTreeMap, HashMap, HashSet, VecDeque}, + collections::{BTreeMap, VecDeque}, future::Future, pin::Pin, task::Poll, @@ -53,7 +53,7 @@ use tokio_util::sync::CancellationToken; use crate::{error::SecondingError, LOG_TARGET}; -use super::{GroupAssignments, PeerData}; +use super::GroupAssignments; /// Candidate supplied with a para head it's built on top of. #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] @@ -228,8 +228,14 @@ pub struct Collations { pub status: CollationStatus, /// Collator we're fetching from, optionally which candidate was requested. /// - /// This is the currently last started fetch, which did not exceed `MAX_UNSHARED_DOWNLOAD_TIME` - /// yet. + /// This is the last fetch for the relay parent. The value is used in + /// `get_next_collation_to_fetch` (called from `dequeue_next_collation_and_fetch`) to determine + /// if the last fetched collation is the same as the one which just finished. If yes - another + /// collation should be fetched. If not - another fetch was already initiated and + /// `get_next_collation_to_fetch` will do nothing. + /// + /// For the reasons above this value is not set to `None` when the fetch is done! Don't use it + /// to check if there is a pending fetch. pub fetching_from: Option<(CollatorId, Option)>, /// Collation that were advertised to us, but we did not yet fetch. Grouped by `ParaId`. waiting_queue: BTreeMap>, @@ -317,38 +323,25 @@ impl Collations { &self, relay_parent_mode: ProspectiveParachainsMode, para_id: ParaId, - peer_data: &HashMap, + num_pending_fetches: usize, ) -> bool { if let ProspectiveParachainsMode::Disabled = relay_parent_mode { // fallback to synchronous backing return self.seconded_count >= 1 } - // All collators in `Collating` state for `para_id` we know about - let collators_for_para = peer_data - .iter() - .filter(|(_, data)| data.collating_para() == Some(para_id)) - .filter_map(|(_, data)| data.collator_id()) - .collect::>(); - - // If there is a pending fetch - count it - let pending_fetch = self - .fetching_from - .as_ref() - .map(|(collator_id, _)| if collators_for_para.contains(&collator_id) { 1 } else { 0 }) - .unwrap_or(0); - // Successful fetches + a pending fetch < claim queue entries for `para_id` let respected_per_para_limit = - self.claims_per_para.get(¶_id).copied().unwrap_or_default() >= - self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + pending_fetch; + self.claims_per_para.get(¶_id).copied().unwrap_or_default() > + self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + + num_pending_fetches; gum::trace!( target: LOG_TARGET, ?para_id, claims_per_para=?self.claims_per_para, fetched_per_para=?self.fetched_per_para, - ?pending_fetch, + ?num_pending_fetches, ?respected_per_para_limit, "is_collations_limit_reached" ); diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 8da0faefe1bd..35d8c57ff5c1 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -1099,7 +1099,7 @@ where if per_relay_parent.collations.is_collations_limit_reached( relay_parent_mode, para_id, - &state.peer_data, + num_pending_collations_for_para_at_relay_parent(&state, para_id, relay_parent), ) { return Err(AdvertisementError::SecondedLimitReached) } @@ -1169,6 +1169,9 @@ where ?relay_parent, "Received advertise collation", ); + + let num_pending_fetches = + num_pending_collations_for_para_at_relay_parent(&state, para_id, relay_parent); let per_relay_parent = match state.per_relay_parent.get_mut(&relay_parent) { Some(rp_state) => rp_state, None => { @@ -1192,7 +1195,7 @@ where }); let collations = &mut per_relay_parent.collations; - if collations.is_collations_limit_reached(relay_parent_mode, para_id, &state.peer_data) { + if collations.is_collations_limit_reached(relay_parent_mode, para_id, num_pending_fetches) { gum::trace!( target: LOG_TARGET, peer_id = ?peer_id, @@ -2103,3 +2106,18 @@ async fn handle_collation_fetch_response( state.metrics.on_request(metrics_result); result } + +// Returns the number of pending fetches for `ParaId` at a specific relay parent. +fn num_pending_collations_for_para_at_relay_parent( + state: &State, + para_id: ParaId, + relay_parent: Hash, +) -> usize { + state + .collation_requests_cancel_handles + .iter() + .filter(|(pending_collation, _)| { + pending_collation.para_id == para_id && pending_collation.relay_parent == relay_parent + }) + .count() +} diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs new file mode 100644 index 000000000000..af4913f2b946 --- /dev/null +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -0,0 +1,61 @@ +use polkadot_node_subsystem_util::runtime::ProspectiveParachainsMode; +use polkadot_primitives::{CollatorId, Id as ParaId}; + +use sp_core::sr25519; + +use super::Collations; + +#[test] +fn cant_add_more_than_claim_queue() { + let para_a = ParaId::from(1); + let para_b = ParaId::from(2); + let assignments = vec![para_a.clone(), para_b.clone(), para_a.clone()]; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + + let mut collations = Collations::new(&assignments); + + // first collation for `para_a` is in the limit + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); + collations.note_fetched(para_a.clone()); + // and `para_b` is not affected + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); + + // second collation for `para_a` is also in the limit + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); + collations.note_fetched(para_a.clone()); + + // `para_b`` is still not affected + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); + + // third collation for `para_a`` will be above the limit + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); + + // one fetch for b + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); + collations.note_fetched(para_b.clone()); + + // and now both paras are over limit + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); +} + +#[test] +fn pending_fetches_are_counted() { + let para_a = ParaId::from(1); + let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); + let para_b = ParaId::from(2); + let assignments = vec![para_a.clone(), para_b.clone(), para_a.clone()]; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + + let mut collations = Collations::new(&assignments); + collations.fetching_from = Some((collator_id_a, None)); + + // first collation for `para_a` is in the limit + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 1)); + collations.note_fetched(para_a.clone()); + + // second collation for `para_a`` is not in the limit due to the pending fetch + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 1)); +} diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 1da035ebc837..edb17d697d78 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -50,6 +50,7 @@ use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt_bad_sig, dummy_hash, }; +mod collation; mod prospective_parachains; const ACTIVITY_TIMEOUT: Duration = Duration::from_millis(500); @@ -83,10 +84,6 @@ struct TestState { impl Default for TestState { fn default() -> Self { - let chain_a = ParaId::from(1); - let chain_b = ParaId::from(2); - - let chain_ids = vec![chain_a, chain_b]; let relay_parent = Hash::repeat_byte(0x05); let collators = iter::repeat(()).map(|_| CollatorPair::generate().0).take(5).collect(); @@ -109,10 +106,16 @@ impl Default for TestState { GroupRotationInfo { session_start_block: 0, group_rotation_frequency: 1, now: 0 }; let cores = vec![ - CoreState::Scheduled(ScheduledCore { para_id: chain_ids[0], collator: None }), + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(Self::CHAIN_IDS[0]), + collator: None, + }), CoreState::Free, CoreState::Occupied(OccupiedCore { - next_up_on_available: Some(ScheduledCore { para_id: chain_ids[1], collator: None }), + next_up_on_available: Some(ScheduledCore { + para_id: ParaId::from(Self::CHAIN_IDS[1]), + collator: None, + }), occupied_since: 0, time_out_at: 1, next_up_on_time_out: None, @@ -121,33 +124,30 @@ impl Default for TestState { candidate_hash: Default::default(), candidate_descriptor: { let mut d = dummy_candidate_descriptor(dummy_hash()); - d.para_id = chain_ids[1]; + d.para_id = ParaId::from(Self::CHAIN_IDS[1]); d }, }), ]; - let async_backing_params = - AsyncBackingParams { max_candidate_depth: 4, allowed_ancestry_len: 3 }; - let mut claim_queue = BTreeMap::new(); claim_queue.insert( CoreIndex(0), - iter::repeat(chain_ids[0]) - .take(async_backing_params.allowed_ancestry_len as usize) + iter::repeat(ParaId::from(Self::CHAIN_IDS[0])) + .take(Self::ASYNC_BACKING_PARAMS.allowed_ancestry_len as usize) .collect(), ); claim_queue.insert(CoreIndex(1), VecDeque::new()); claim_queue.insert( CoreIndex(2), - iter::repeat(chain_ids[1]) - .take(async_backing_params.allowed_ancestry_len as usize) + iter::repeat(ParaId::from(Self::CHAIN_IDS[1])) + .take(Self::ASYNC_BACKING_PARAMS.allowed_ancestry_len as usize) .collect(), ); Self { - chain_ids, + chain_ids: Self::CHAIN_IDS.map(|id| ParaId::from(id)).to_vec(), relay_parent, collators, validator_public, @@ -155,11 +155,52 @@ impl Default for TestState { group_rotation_info, cores, claim_queue, - async_backing_params, + async_backing_params: Self::ASYNC_BACKING_PARAMS, } } } +impl TestState { + const CHAIN_IDS: [u32; 2] = [1, 2]; + const ASYNC_BACKING_PARAMS: AsyncBackingParams = + AsyncBackingParams { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + + fn with_shared_core() -> Self { + let mut state = Self::default(); + + let cores = vec![ + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(Self::CHAIN_IDS[0]), + collator: None, + }), + CoreState::Free, + ]; + + let mut claim_queue = BTreeMap::new(); + claim_queue.insert( + CoreIndex(0), + VecDeque::from_iter( + [ + ParaId::from(Self::CHAIN_IDS[0]), + ParaId::from(Self::CHAIN_IDS[1]), + ParaId::from(Self::CHAIN_IDS[0]), + ] + .into_iter(), + ), + ); + + assert!( + claim_queue.get(&CoreIndex(0)).unwrap().len() == + Self::ASYNC_BACKING_PARAMS.allowed_ancestry_len as usize + ); + + state.cores = cores; + state.claim_queue = claim_queue; + + state + } +} + type VirtualOverseer = polkadot_node_subsystem_test_helpers::TestSubsystemContextHandle; diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index cd3b4ec8d054..c92013669987 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -325,6 +325,83 @@ async fn assert_persisted_validation_data( } } +async fn submit_second_and_assert( + virtual_overseer: &mut VirtualOverseer, + keystore: KeystorePtr, + para_id: ParaId, + relay_parent: Hash, + collator: PeerId, + candidate_head_data: HeadData, +) { + let mut candidate = dummy_candidate_receipt_bad_sig(relay_parent, Some(Default::default())); + candidate.descriptor.para_id = para_id; + candidate.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); + let commitments = CandidateCommitments { + head_data: candidate_head_data, + horizontal_messages: Default::default(), + upward_messages: Default::default(), + new_validation_code: None, + processed_downward_messages: 0, + hrmp_watermark: 0, + }; + candidate.commitments_hash = commitments.hash(); + + let candidate_hash = candidate.hash(); + let parent_head_data_hash = Hash::zero(); + + advertise_collation( + virtual_overseer, + collator, + relay_parent, + Some((candidate_hash, parent_head_data_hash)), + ) + .await; + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::CandidateBacking( + CandidateBackingMessage::CanSecond(request, tx), + ) => { + assert_eq!(request.candidate_hash, candidate_hash); + assert_eq!(request.candidate_para_id, para_id); + assert_eq!(request.parent_head_data_hash, parent_head_data_hash); + tx.send(true).expect("receiving side should be alive"); + } + ); + + let response_channel = assert_fetch_collation_request( + virtual_overseer, + relay_parent, + para_id, + Some(candidate_hash), + ) + .await; + + let pov = PoV { block_data: BlockData(vec![1]) }; + + response_channel + .send(Ok(( + request_v2::CollationFetchingResponse::Collation(candidate.clone(), pov.clone()) + .encode(), + ProtocolName::from(""), + ))) + .expect("Sending response should succeed"); + + assert_candidate_backing_second( + virtual_overseer, + relay_parent, + para_id, + &pov, + CollationVersion::V2, + ) + .await; + + let candidate = CommittedCandidateReceipt { descriptor: candidate.descriptor, commitments }; + + send_seconded_statement(virtual_overseer, keystore.clone(), &candidate).await; + + assert_collation_seconded(virtual_overseer, relay_parent, collator, CollationVersion::V2).await; +} + #[test] fn v1_advertisement_accepted_and_seconded() { let test_state = TestState::default(); @@ -609,79 +686,17 @@ fn second_multiple_candidates_per_relay_parent() { ) .await; + // `allowed_ancestry_len` equals the size of the claim queue for i in 0..(test_state.async_backing_params.allowed_ancestry_len) { - let mut candidate = dummy_candidate_receipt_bad_sig(head_c, Some(Default::default())); - candidate.descriptor.para_id = test_state.chain_ids[0]; - candidate.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); - let commitments = CandidateCommitments { - head_data: HeadData(vec![i as u8]), - horizontal_messages: Default::default(), - upward_messages: Default::default(), - new_validation_code: None, - processed_downward_messages: 0, - hrmp_watermark: 0, - }; - candidate.commitments_hash = commitments.hash(); - - let candidate_hash = candidate.hash(); - let parent_head_data_hash = Hash::zero(); - - advertise_collation( - &mut virtual_overseer, - peer_a, - head_c, - Some((candidate_hash, parent_head_data_hash)), - ) - .await; - assert_matches!( - overseer_recv(&mut virtual_overseer).await, - AllMessages::CandidateBacking( - CandidateBackingMessage::CanSecond(request, tx), - ) => { - assert_eq!(request.candidate_hash, candidate_hash); - assert_eq!(request.candidate_para_id, test_state.chain_ids[0]); - assert_eq!(request.parent_head_data_hash, parent_head_data_hash); - tx.send(true).expect("receiving side should be alive"); - } - ); - - let response_channel = assert_fetch_collation_request( - &mut virtual_overseer, - head_c, - test_state.chain_ids[0], - Some(candidate_hash), - ) - .await; - - let pov = PoV { block_data: BlockData(vec![1]) }; - - response_channel - .send(Ok(( - request_v2::CollationFetchingResponse::Collation( - candidate.clone(), - pov.clone(), - ) - .encode(), - ProtocolName::from(""), - ))) - .expect("Sending response should succeed"); - - assert_candidate_backing_second( + submit_second_and_assert( &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[0]), head_c, - test_state.chain_ids[0], - &pov, - CollationVersion::V2, + peer_a, + HeadData(vec![i as u8]), ) .await; - - let candidate = - CommittedCandidateReceipt { descriptor: candidate.descriptor, commitments }; - - send_seconded_statement(&mut virtual_overseer, keystore.clone(), &candidate).await; - - assert_collation_seconded(&mut virtual_overseer, head_c, peer_a, CollationVersion::V2) - .await; } // No more advertisements can be made for this relay parent. @@ -1358,3 +1373,217 @@ fn child_blocked_from_seconding_by_parent(#[case] valid_parent: bool) { virtual_overseer }); } + +#[test] +fn collations_outside_limits_are_not_fetched() { + let test_state = TestState::with_shared_core(); + + test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { + let TestHarness { mut virtual_overseer, keystore } = test_harness; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 2; + + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; + + let peer_a = PeerId::random(); + let pair_a = CollatorPair::generate().0; + + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair_a.clone(), + test_state.chain_ids[0], + CollationVersion::V2, + ) + .await; + + let peer_b = PeerId::random(); + let pair_b = CollatorPair::generate().0; + + connect_and_declare_collator( + &mut virtual_overseer, + peer_b, + pair_b.clone(), + test_state.chain_ids[1], + CollationVersion::V2, + ) + .await; + + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[0]), + head_b, + peer_a, + HeadData(vec![1 as u8]), + ) + .await; + + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[1]), + head_b, + peer_b, + HeadData(vec![2 as u8]), + ) + .await; + + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[0]), + head_b, + peer_a, + HeadData(vec![3 as u8]), + ) + .await; + + // No more advertisements can be made for this relay parent. + + // verify for peer_a + let candidate_hash = CandidateHash(Hash::repeat_byte(0xAA)); + advertise_collation( + &mut virtual_overseer, + peer_a, + head_b, + Some((candidate_hash, Hash::zero())), + ) + .await; + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + // verify for peer_b + let candidate_hash = CandidateHash(Hash::repeat_byte(0xBB)); + advertise_collation( + &mut virtual_overseer, + peer_b, + head_b, + Some((candidate_hash, Hash::zero())), + ) + .await; + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + virtual_overseer + }); +} + +#[test] +fn fair_collation_fetches() { + let test_state = TestState::with_shared_core(); + + test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { + let TestHarness { mut virtual_overseer, keystore } = test_harness; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 2; + + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; + + let peer_a = PeerId::random(); + let pair_a = CollatorPair::generate().0; + + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair_a.clone(), + test_state.chain_ids[0], + CollationVersion::V2, + ) + .await; + + let peer_b = PeerId::random(); + let pair_b = CollatorPair::generate().0; + + connect_and_declare_collator( + &mut virtual_overseer, + peer_b, + pair_b.clone(), + test_state.chain_ids[1], + CollationVersion::V2, + ) + .await; + + // `peer_a` sends two advertisements (its claim queue limit) + for i in 0..2u8 { + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[0]), + head_b, + peer_a, + HeadData(vec![i]), + ) + .await; + } + + // `peer_a` sends another advertisement and it is ignored + let candidate_hash = CandidateHash(Hash::repeat_byte(0xAA)); + advertise_collation( + &mut virtual_overseer, + peer_a, + head_b, + Some((candidate_hash, Hash::zero())), + ) + .await; + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + // `peer_b` should still be able to advertise its collation + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[1]), + head_b, + peer_b, + HeadData(vec![0 as u8]), + ) + .await; + + // And no more advertisements can be made for this relay parent. + + // verify for peer_a + let candidate_hash = CandidateHash(Hash::repeat_byte(0xBB)); + advertise_collation( + &mut virtual_overseer, + peer_a, + head_b, + Some((candidate_hash, Hash::zero())), + ) + .await; + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + // verify for peer_b + let candidate_hash = CandidateHash(Hash::repeat_byte(0xCC)); + advertise_collation( + &mut virtual_overseer, + peer_b, + head_b, + Some((candidate_hash, Hash::zero())), + ) + .await; + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + virtual_overseer + }); +} From c07890b0bef348188ab58c7022c1f0109157a40b Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 1 Jul 2024 14:54:07 +0300 Subject: [PATCH 09/55] Remove unnecessary clone --- .../src/validator_side/tests/collation.rs | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index af4913f2b946..256aeea3e819 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -9,35 +9,35 @@ use super::Collations; fn cant_add_more_than_claim_queue() { let para_a = ParaId::from(1); let para_b = ParaId::from(2); - let assignments = vec![para_a.clone(), para_b.clone(), para_a.clone()]; + let assignments = vec![para_a, para_b, para_a]; let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; let mut collations = Collations::new(&assignments); // first collation for `para_a` is in the limit - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); - collations.note_fetched(para_a.clone()); + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); + collations.note_fetched(para_a); // and `para_b` is not affected - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); // second collation for `para_a` is also in the limit - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); - collations.note_fetched(para_a.clone()); + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); + collations.note_fetched(para_a); // `para_b`` is still not affected - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); // third collation for `para_a`` will be above the limit - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); // one fetch for b - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); - collations.note_fetched(para_b.clone()); + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); + collations.note_fetched(para_b); // and now both paras are over limit - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 0)); - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_b.clone(), 0)); + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); } #[test] @@ -45,7 +45,7 @@ fn pending_fetches_are_counted() { let para_a = ParaId::from(1); let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); let para_b = ParaId::from(2); - let assignments = vec![para_a.clone(), para_b.clone(), para_a.clone()]; + let assignments = vec![para_a, para_b, para_a]; let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; @@ -53,9 +53,9 @@ fn pending_fetches_are_counted() { collations.fetching_from = Some((collator_id_a, None)); // first collation for `para_a` is in the limit - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 1)); - collations.note_fetched(para_a.clone()); + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 1)); + collations.note_fetched(para_a); // second collation for `para_a`` is not in the limit due to the pending fetch - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a.clone(), 1)); + assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a, 1)); } From e50440e11430baae0ff6fe605f1e998cdbbca55a Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 1 Jul 2024 15:20:23 +0300 Subject: [PATCH 10/55] Comments --- .../collator-protocol/src/validator_side/collation.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 629a4eeb1305..012afd43df24 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -306,7 +306,9 @@ impl Collations { self.status.back_to_waiting(relay_parent_mode); match self.status { - // We don't need to fetch any other collation when we already have seconded one. + // If async backing is enabled `back_to_waiting` will change `Seconded` state to + // `Waiting` so that we can fetch more collations. If async backing is disabled we can't + // fetch more than one collation per relay parent so `None` is returned. CollationStatus::Seconded => None, CollationStatus::Waiting => self.pick_a_collation_to_fetch(&assignments.current), CollationStatus::WaitingOnValidation | CollationStatus::Fetching => @@ -330,7 +332,7 @@ impl Collations { return self.seconded_count >= 1 } - // Successful fetches + a pending fetch < claim queue entries for `para_id` + // Successful fetches + pending fetches < claim queue entries for `para_id` let respected_per_para_limit = self.claims_per_para.get(¶_id).copied().unwrap_or_default() > self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + @@ -412,7 +414,7 @@ impl Collations { match collations.pop_front() { Some(collation) => return Some(collation), None => { - unreachable!("Collation can't be empty!") + unreachable!("Collation can't be empty because empty ones are skipped at the beginning of the loop.") }, } } From 42b05c73c3f560b7e28aa4a186f461f593202350 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 1 Jul 2024 17:59:22 +0300 Subject: [PATCH 11/55] Better var names --- .../src/validator_side/collation.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 012afd43df24..fd1c8e4cab72 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -53,8 +53,6 @@ use tokio_util::sync::CancellationToken; use crate::{error::SecondingError, LOG_TARGET}; -use super::GroupAssignments; - /// Candidate supplied with a para head it's built on top of. #[derive(Debug, Copy, Clone, Hash, Eq, PartialEq)] pub struct ProspectiveCandidate { @@ -249,9 +247,9 @@ pub struct Collations { } impl Collations { - pub(super) fn new(assignments: &Vec) -> Self { + pub(super) fn new(claim_queue: &Vec) -> Self { let mut claims_per_para = BTreeMap::new(); - for para_id in assignments { + for para_id in claim_queue { *claims_per_para.entry(*para_id).or_default() += 1; } @@ -285,7 +283,7 @@ impl Collations { &mut self, finished_one: &(CollatorId, Option), relay_parent_mode: ProspectiveParachainsMode, - assignments: &GroupAssignments, + claim_queue: &Vec, ) -> Option<(PendingCollation, CollatorId)> { // If finished one does not match waiting_collation, then we already dequeued another fetch // to replace it. @@ -310,7 +308,7 @@ impl Collations { // `Waiting` so that we can fetch more collations. If async backing is disabled we can't // fetch more than one collation per relay parent so `None` is returned. CollationStatus::Seconded => None, - CollationStatus::Waiting => self.pick_a_collation_to_fetch(&assignments.current), + CollationStatus::Waiting => self.pick_a_collation_to_fetch(&claim_queue), CollationStatus::WaitingOnValidation | CollationStatus::Fetching => unreachable!("We have reset the status above!"), } @@ -373,12 +371,12 @@ impl Collations { /// picked. fn pick_a_collation_to_fetch( &mut self, - claims: &Vec, + claim_queue: &Vec, ) -> Option<(PendingCollation, CollatorId)> { gum::trace!( target: LOG_TARGET, waiting_queue = ?self.waiting_queue, - claim_queue = ?claims, + ?claim_queue, "Pick a collation to fetch." ); @@ -408,7 +406,7 @@ impl Collations { } if let Some((_, mut lowest_score)) = lowest_score { - for claim in claims { + for claim in claim_queue { if let Some((_, collations)) = lowest_score.iter_mut().find(|(id, _)| *id == claim) { match collations.pop_front() { From 2f5a466b7ba1a6820e7c63089b090e95838f4e67 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 2 Jul 2024 10:39:00 +0300 Subject: [PATCH 12/55] Fix `pick_a_collation_to_fetch` and add more tests --- .../src/validator_side/collation.rs | 11 ++- .../src/validator_side/mod.rs | 2 +- .../src/validator_side/tests/collation.rs | 98 +++++++++++++++++++ 3 files changed, 107 insertions(+), 4 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index fd1c8e4cab72..26641a30671c 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -375,7 +375,9 @@ impl Collations { ) -> Option<(PendingCollation, CollatorId)> { gum::trace!( target: LOG_TARGET, - waiting_queue = ?self.waiting_queue, + waiting_queue=?self.waiting_queue, + fetched_per_para=?self.fetched_per_para, + claims_per_para=?self.claims_per_para, ?claim_queue, "Pick a collation to fetch." ); @@ -387,8 +389,11 @@ impl Collations { .fetched_per_para .get(para_id) .copied() - .unwrap_or_default() - .saturating_div(self.claims_per_para.get(para_id).copied().unwrap_or_default()); + .map(|v| { + (v as f64) / + (self.claims_per_para.get(para_id).copied().unwrap_or_default() as f64) + }) + .unwrap_or_default(); // skip empty queues if collations.is_empty() { diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 35d8c57ff5c1..437825033dc8 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -1732,7 +1732,7 @@ async fn dequeue_next_collation_and_fetch( state.collations.get_next_collation_to_fetch( &previous_fetch, state.prospective_parachains_mode, - &state.assignment, + &state.assignment.current, ) }) { gum::debug!( diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 256aeea3e819..2bd0e587229e 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -59,3 +59,101 @@ fn pending_fetches_are_counted() { // second collation for `para_a`` is not in the limit due to the pending fetch assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a, 1)); } + +#[test] +fn collation_fetching_respects_claim_queue() { + let para_a = ParaId::from(1); + let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); + let peer_a = PeerId::random(); + + let para_b = ParaId::from(2); + let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); + let peer_b = PeerId::random(); + + let assignments = vec![para_a, para_b, para_a]; + let mut collations = Collations::new(&assignments); + collations.fetching_from = None; + + let relay_parent = Hash::repeat_byte(0x01); + + let collation_a1 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(1)), + parent_head_data_hash: Hash::repeat_byte(1), + }), + ), + collator_id_a.clone(), + ); + + let collation_a2 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(2)), + parent_head_data_hash: Hash::repeat_byte(2), + }), + ), + collator_id_a.clone(), + ); + + let collation_b1 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(3)), + parent_head_data_hash: Hash::repeat_byte(3), + }), + ), + collator_id_b.clone(), + ); + + collations.add_to_waiting_queue(collation_a1.clone()); + collations.add_to_waiting_queue(collation_a2.clone()); + collations.add_to_waiting_queue(collation_b1.clone()); + + let claim_queue = vec![para_a, para_b, para_a]; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + + assert_eq!( + Some(collation_a1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a1.0.para_id); + + assert_eq!( + Some(collation_b1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b1.0.para_id); + + assert_eq!( + Some(collation_a2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a2.0.para_id); +} + From ff96ef9651327f1bf3c287ffbe65bec04d422573 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 2 Jul 2024 13:46:22 +0300 Subject: [PATCH 13/55] Fix test: `collation_fetching_respects_claim_queue` --- .../src/validator_side/tests/collation.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 2bd0e587229e..e34617b8d4ee 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -1,9 +1,10 @@ use polkadot_node_subsystem_util::runtime::ProspectiveParachainsMode; -use polkadot_primitives::{CollatorId, Id as ParaId}; +use polkadot_primitives::{CandidateHash, CollatorId, Hash, Id as ParaId}; +use sc_network::PeerId; use sp_core::sr25519; -use super::Collations; +use super::{Collations, PendingCollation, ProspectiveCandidate}; #[test] fn cant_add_more_than_claim_queue() { @@ -70,8 +71,11 @@ fn collation_fetching_respects_claim_queue() { let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); let peer_b = PeerId::random(); - let assignments = vec![para_a, para_b, para_a]; - let mut collations = Collations::new(&assignments); + let claim_queue = vec![para_a, para_b, para_a]; + let mut collations = Collations::new(&claim_queue); + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + collations.fetching_from = None; let relay_parent = Hash::repeat_byte(0x01); @@ -119,10 +123,6 @@ fn collation_fetching_respects_claim_queue() { collations.add_to_waiting_queue(collation_a2.clone()); collations.add_to_waiting_queue(collation_b1.clone()); - let claim_queue = vec![para_a, para_b, para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; - assert_eq!( Some(collation_a1.clone()), collations.get_next_collation_to_fetch( @@ -156,4 +156,3 @@ fn collation_fetching_respects_claim_queue() { ); collations.note_fetched(collation_a2.0.para_id); } - From e837689fcfd168a2f20de4eda7ca6f863981bb66 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 2 Jul 2024 14:33:41 +0300 Subject: [PATCH 14/55] Add `collation_fetching_fallback_works` test + comments --- Cargo.lock | 1 + .../node/network/collator-protocol/Cargo.toml | 1 + .../src/validator_side/collation.rs | 26 ++++--- .../src/validator_side/tests/collation.rs | 75 +++++++++++++++++++ 4 files changed, 94 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbb785a618a8..30343e645deb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12818,6 +12818,7 @@ dependencies = [ "sp-keyring", "sp-keystore", "sp-runtime", + "sp-tracing 16.0.0", "thiserror", "tokio-util", "tracing-gum", diff --git a/polkadot/node/network/collator-protocol/Cargo.toml b/polkadot/node/network/collator-protocol/Cargo.toml index a56c1c7dfe98..c43b4dac2d5e 100644 --- a/polkadot/node/network/collator-protocol/Cargo.toml +++ b/polkadot/node/network/collator-protocol/Cargo.toml @@ -39,6 +39,7 @@ sp-keyring = { path = "../../../../substrate/primitives/keyring" } sc-keystore = { path = "../../../../substrate/client/keystore" } sc-network = { path = "../../../../substrate/client/network" } codec = { package = "parity-scale-codec", version = "3.6.12", features = ["std"] } +sp-tracing = { path = "../../../../substrate/primitives/tracing"} polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 26641a30671c..a577eda18e1b 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -242,14 +242,22 @@ pub struct Collations { /// What collations were fetched so far for this relay parent. fetched_per_para: BTreeMap, // Claims per `ParaId` for the assigned core at the relay parent. This information is obtained - // from the claim queue. + // from `GroupAssignments` which contains either the claim queue for the core or the `ParaId` + // of the parachain assigned to the core. claims_per_para: BTreeMap, } impl Collations { - pub(super) fn new(claim_queue: &Vec) -> Self { + /// `Collations` should work with and without claim queue support. To make this happen without + /// creating two parallel implementations instead of working with the claim queue directly it + /// uses `GroupAssignments`. If the runtime supports claim queue `GroupAssignments` contains the + /// claim queue for the core assigned to the (backing group of the) validator. If the runtime + /// doesn't support claim queue `GroupAssignments` contains only one entry - the `ParaId` of the + /// parachain assigned to the core. This way we can handle both cases with a single + /// implementation and avoid code duplication. + pub(super) fn new(group_assignments: &Vec) -> Self { let mut claims_per_para = BTreeMap::new(); - for para_id in claim_queue { + for para_id in group_assignments { *claims_per_para.entry(*para_id).or_default() += 1; } @@ -283,7 +291,7 @@ impl Collations { &mut self, finished_one: &(CollatorId, Option), relay_parent_mode: ProspectiveParachainsMode, - claim_queue: &Vec, + group_assignments: &Vec, ) -> Option<(PendingCollation, CollatorId)> { // If finished one does not match waiting_collation, then we already dequeued another fetch // to replace it. @@ -308,14 +316,14 @@ impl Collations { // `Waiting` so that we can fetch more collations. If async backing is disabled we can't // fetch more than one collation per relay parent so `None` is returned. CollationStatus::Seconded => None, - CollationStatus::Waiting => self.pick_a_collation_to_fetch(&claim_queue), + CollationStatus::Waiting => self.pick_a_collation_to_fetch(&group_assignments), CollationStatus::WaitingOnValidation | CollationStatus::Fetching => unreachable!("We have reset the status above!"), } } /// Checks if another collation can be accepted. The number of collations that can be fetched - /// per parachain is limited by the entries in the claim queue for the `ParaId` in question. + /// per parachain is limited by the entries in claim queue for the `ParaId` in question. /// /// If prospective parachains mode is not enabled then we fall back to synchronous backing. In /// this case there is a limit of 1 collation per relay parent. @@ -371,14 +379,14 @@ impl Collations { /// picked. fn pick_a_collation_to_fetch( &mut self, - claim_queue: &Vec, + group_assignments: &Vec, ) -> Option<(PendingCollation, CollatorId)> { gum::trace!( target: LOG_TARGET, waiting_queue=?self.waiting_queue, fetched_per_para=?self.fetched_per_para, claims_per_para=?self.claims_per_para, - ?claim_queue, + ?group_assignments, "Pick a collation to fetch." ); @@ -411,7 +419,7 @@ impl Collations { } if let Some((_, mut lowest_score)) = lowest_score { - for claim in claim_queue { + for claim in group_assignments { if let Some((_, collations)) = lowest_score.iter_mut().find(|(id, _)| *id == claim) { match collations.pop_front() { diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index e34617b8d4ee..6ca2d2075065 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -8,6 +8,8 @@ use super::{Collations, PendingCollation, ProspectiveCandidate}; #[test] fn cant_add_more_than_claim_queue() { + sp_tracing::init_for_tests(); + let para_a = ParaId::from(1); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; @@ -43,6 +45,8 @@ fn cant_add_more_than_claim_queue() { #[test] fn pending_fetches_are_counted() { + sp_tracing::init_for_tests(); + let para_a = ParaId::from(1); let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); let para_b = ParaId::from(2); @@ -63,6 +67,8 @@ fn pending_fetches_are_counted() { #[test] fn collation_fetching_respects_claim_queue() { + sp_tracing::init_for_tests(); + let para_a = ParaId::from(1); let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); let peer_a = PeerId::random(); @@ -156,3 +162,72 @@ fn collation_fetching_respects_claim_queue() { ); collations.note_fetched(collation_a2.0.para_id); } + +#[test] +fn collation_fetching_fallback_works() { + sp_tracing::init_for_tests(); + + let para_a = ParaId::from(1); + let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); + let peer_a = PeerId::random(); + + let claim_queue = vec![para_a]; + let mut collations = Collations::new(&claim_queue); + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + + collations.fetching_from = None; + + let relay_parent = Hash::repeat_byte(0x01); + + let collation_a1 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(1)), + parent_head_data_hash: Hash::repeat_byte(1), + }), + ), + collator_id_a.clone(), + ); + + let collation_a2 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(2)), + parent_head_data_hash: Hash::repeat_byte(2), + }), + ), + collator_id_a.clone(), + ); + + collations.add_to_waiting_queue(collation_a1.clone()); + collations.add_to_waiting_queue(collation_a2.clone()); + + assert_eq!( + Some(collation_a1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a1.0.para_id); + + assert_eq!( + Some(collation_a2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a2.0.para_id); +} From 91cdd134532d8ddbd190506c5a6f1907f75f5c2c Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 3 Jul 2024 11:08:17 +0300 Subject: [PATCH 15/55] More tests --- .../src/validator_side/tests/collation.rs | 124 ++++++++++++++++++ .../src/validator_side/tests/mod.rs | 31 ++++- .../tests/prospective_parachains.rs | 120 +++++++++++++++-- 3 files changed, 261 insertions(+), 14 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 6ca2d2075065..cdc1cda070f2 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -231,3 +231,127 @@ fn collation_fetching_fallback_works() { ); collations.note_fetched(collation_a2.0.para_id); } + +#[test] +fn collation_fetching_prefer_entries_earlier_in_claim_queue() { + sp_tracing::init_for_tests(); + + let para_a = ParaId::from(1); + let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); + let peer_a = PeerId::random(); + + let para_b = ParaId::from(2); + let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); + let peer_b = PeerId::random(); + + let claim_queue = vec![para_a, para_b, para_a, para_b]; + let mut collations = Collations::new(&claim_queue); + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 5, allowed_ancestry_len: 4 }; + + collations.fetching_from = None; + + let relay_parent = Hash::repeat_byte(0x01); + + let collation_a1 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(1)), + parent_head_data_hash: Hash::repeat_byte(1), + }), + ), + collator_id_a.clone(), + ); + + let collation_a2 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(2)), + parent_head_data_hash: Hash::repeat_byte(2), + }), + ), + collator_id_a.clone(), + ); + + let collation_b1 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(3)), + parent_head_data_hash: Hash::repeat_byte(3), + }), + ), + collator_id_b.clone(), + ); + + let collation_b2 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(4)), + parent_head_data_hash: Hash::repeat_byte(4), + }), + ), + collator_id_b.clone(), + ); + + // Despite the order here the fetches should be a1, b1, a2, b2 + collations.add_to_waiting_queue(collation_b1.clone()); + collations.add_to_waiting_queue(collation_b2.clone()); + collations.add_to_waiting_queue(collation_a1.clone()); + collations.add_to_waiting_queue(collation_a2.clone()); + + assert_eq!( + Some(collation_a1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a1.0.para_id); + + assert_eq!( + Some(collation_b1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b1.0.para_id); + + assert_eq!( + Some(collation_a2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a2.0.para_id); + + assert_eq!( + Some(collation_b2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b2.0.para_id); +} diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index edb17d697d78..7243af9fe4ca 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -78,7 +78,7 @@ struct TestState { validator_groups: Vec>, group_rotation_info: GroupRotationInfo, cores: Vec, - claim_queue: BTreeMap>, + claim_queue: Option>>, async_backing_params: AsyncBackingParams, } @@ -154,7 +154,7 @@ impl Default for TestState { validator_groups, group_rotation_info, cores, - claim_queue, + claim_queue: Some(claim_queue), async_backing_params: Self::ASYNC_BACKING_PARAMS, } } @@ -195,7 +195,21 @@ impl TestState { ); state.cores = cores; - state.claim_queue = claim_queue; + state.claim_queue = Some(claim_queue); + + state + } + + fn without_claim_queue() -> Self { + let mut state = Self::default(); + state.claim_queue = None; + state.cores = vec![ + CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(Self::CHAIN_IDS[0]), + collator: None, + }), + CoreState::Free, + ]; state } @@ -350,7 +364,16 @@ async fn respond_to_core_info_queries( _, RuntimeApiRequest::ClaimQueue(tx), )) => { - let _ = tx.send(Ok(test_state.claim_queue.clone())); + match test_state.claim_queue { + Some(ref claim_queue) => { + let _ = tx.send(Ok(claim_queue.clone())); + }, + None => { + let _ = tx.send(Err(RuntimeApiError::NotSupported { runtime_api_name: "doesnt_matter" })); + } + + } + } ); } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index c92013669987..70d189334a95 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -76,19 +76,28 @@ async fn assert_assign_incoming( parent, RuntimeApiRequest::Version(tx), )) if parent == hash => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); + match test_state.claim_queue { + Some(_) => { + let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); + }, + None => { + let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT - 1)); + } + } } ); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - parent, - RuntimeApiRequest::ClaimQueue(tx), - )) if parent == hash => { - let _ = tx.send(Ok(test_state.claim_queue.clone())); - } - ); + if let Some(claim_queue) = &test_state.claim_queue { + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + parent, + RuntimeApiRequest::ClaimQueue(tx), + )) if parent == hash => { + let _ = tx.send(Ok(claim_queue.clone())); + } + ); + } } /// Handle a view update. @@ -1587,3 +1596,94 @@ fn fair_collation_fetches() { virtual_overseer }); } + +#[test] +fn collation_fetches_without_claimqueue() { + let test_state = TestState::without_claim_queue(); + + test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { + let TestHarness { mut virtual_overseer, keystore } = test_harness; + + // Grandparent of head `a`. + let head_b = Hash::from_low_u64_be(128); + let head_b_num: u32 = 2; + + update_view( + &mut virtual_overseer, + &test_state, + vec![(head_b, head_b_num)], + 1, + &test_state.async_backing_params, + ) + .await; + + let peer_a = PeerId::random(); + let pair_a = CollatorPair::generate().0; + + connect_and_declare_collator( + &mut virtual_overseer, + peer_a, + pair_a.clone(), + test_state.chain_ids[0], + CollationVersion::V2, + ) + .await; + + let peer_b = PeerId::random(); + let pair_b = CollatorPair::generate().0; + + // connect an unneeded collator + connect_and_declare_collator( + &mut virtual_overseer, + peer_b, + pair_b.clone(), + test_state.chain_ids[1], + CollationVersion::V2, + ) + .await; + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::ReportPeer(ReportPeerMessage::Single(peer_id, _)), + ) => { + assert_eq!(peer_id, peer_b); + } + ); + assert_matches!( + overseer_recv(&mut virtual_overseer).await, + AllMessages::NetworkBridgeTx( + NetworkBridgeTxMessage::DisconnectPeer(peer_id, peer_set) + ) => { + assert_eq!(peer_id, peer_b); + assert_eq!(peer_set, PeerSet::Collation); + } + ); + + // in fallback mode up to `max_candidate_depth` collations are accepted + for i in 0..test_state.async_backing_params.max_candidate_depth { + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[0]), + head_b, + peer_a, + HeadData(vec![i as u8]), + ) + .await; + } + + // `peer_a` sends another advertisement and it is ignored + let candidate_hash = CandidateHash(Hash::repeat_byte(0xAA)); + advertise_collation( + &mut virtual_overseer, + peer_a, + head_b, + Some((candidate_hash, Hash::zero())), + ) + .await; + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + virtual_overseer + }); +} From 9f2d59be85f8af3b897c0d1c5dc18206717f5c1f Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 3 Jul 2024 11:57:06 +0300 Subject: [PATCH 16/55] Fix collation limit fallback --- .../tests/prospective_parachains.rs | 20 +++++++ .../src/validator_side/collation.rs | 60 ++++++++++++------- .../src/validator_side/tests/collation.rs | 36 +++++++---- .../tests/prospective_parachains.rs | 19 +++++- .../node/subsystem-util/src/runtime/mod.rs | 28 +++++++-- 5 files changed, 127 insertions(+), 36 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs index ea8fdb0e04fb..9e68247d5b39 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs @@ -58,6 +58,16 @@ async fn update_view( } ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, + RuntimeApiRequest::Version(tx), + )) => { + let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); + } + ); + let min_number = leaf_number.saturating_sub(ASYNC_BACKING_PARAMETERS.allowed_ancestry_len); let ancestry_len = leaf_number + 1 - min_number; @@ -96,6 +106,16 @@ async fn update_view( } ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, + RuntimeApiRequest::Version(tx), + )) => { + let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); + } + ); + assert_matches!( overseer_recv_with_timeout(virtual_overseer, Duration::from_millis(50)).await.unwrap(), AllMessages::RuntimeApi( diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index a577eda18e1b..6c55c92681eb 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -327,34 +327,50 @@ impl Collations { /// /// If prospective parachains mode is not enabled then we fall back to synchronous backing. In /// this case there is a limit of 1 collation per relay parent. + /// + /// If prospective parachains mode is enabled but claim queue is not supported then up to + /// `max_candidate_depth + 1` seconded collations are supported. In theory in this case if two + /// parachains are sharing a core no fairness is guaranteed between them and the faster one can + /// starve the slower one by exhausting the limit with its own advertisements. In practice this + /// should not happen because core sharing implies core time support which implies the claim + /// queue is available. pub(super) fn is_collations_limit_reached( &self, relay_parent_mode: ProspectiveParachainsMode, para_id: ParaId, num_pending_fetches: usize, ) -> bool { - if let ProspectiveParachainsMode::Disabled = relay_parent_mode { - // fallback to synchronous backing - return self.seconded_count >= 1 - } + match relay_parent_mode { + ProspectiveParachainsMode::Disabled => return self.seconded_count >= 1, + ProspectiveParachainsMode::Enabled { + max_candidate_depth, + allowed_ancestry_len: _, + claim_queue_support, + } if !claim_queue_support => self.seconded_count >= max_candidate_depth + 1, + ProspectiveParachainsMode::Enabled { + max_candidate_depth: _, + allowed_ancestry_len: _, + claim_queue_support: _, + } => { + // Successful fetches + pending fetches < claim queue entries for `para_id` + let respected_per_para_limit = + self.claims_per_para.get(¶_id).copied().unwrap_or_default() > + self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + + num_pending_fetches; - // Successful fetches + pending fetches < claim queue entries for `para_id` - let respected_per_para_limit = - self.claims_per_para.get(¶_id).copied().unwrap_or_default() > - self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + - num_pending_fetches; - - gum::trace!( - target: LOG_TARGET, - ?para_id, - claims_per_para=?self.claims_per_para, - fetched_per_para=?self.fetched_per_para, - ?num_pending_fetches, - ?respected_per_para_limit, - "is_collations_limit_reached" - ); + gum::trace!( + target: LOG_TARGET, + ?para_id, + claims_per_para=?self.claims_per_para, + fetched_per_para=?self.fetched_per_para, + ?num_pending_fetches, + ?respected_per_para_limit, + "is_collations_limit_reached" + ); - !respected_per_para_limit + !respected_per_para_limit + }, + } } /// Adds a new collation to the waiting queue for the relay parent. This function doesn't @@ -377,6 +393,10 @@ impl Collations { /// /// If two parachains has got the same score the one which is earlier in the claim queue will be /// picked. + /// + /// If claim queue is not supported then `group_assignment` should contain just one element and + /// the score won't matter. In this case collations will be fetched in the order they were + /// received. fn pick_a_collation_to_fetch( &mut self, group_assignments: &Vec, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index cdc1cda070f2..11b12a52efdf 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -13,8 +13,11 @@ fn cant_add_more_than_claim_queue() { let para_a = ParaId::from(1); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let relay_parent_mode = ProspectiveParachainsMode::Enabled { + max_candidate_depth: 4, + allowed_ancestry_len: 3, + claim_queue_support: true, + }; let mut collations = Collations::new(&assignments); @@ -51,8 +54,11 @@ fn pending_fetches_are_counted() { let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let relay_parent_mode = ProspectiveParachainsMode::Enabled { + max_candidate_depth: 4, + allowed_ancestry_len: 3, + claim_queue_support: true, + }; let mut collations = Collations::new(&assignments); collations.fetching_from = Some((collator_id_a, None)); @@ -79,8 +85,11 @@ fn collation_fetching_respects_claim_queue() { let claim_queue = vec![para_a, para_b, para_a]; let mut collations = Collations::new(&claim_queue); - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let relay_parent_mode = ProspectiveParachainsMode::Enabled { + max_candidate_depth: 4, + allowed_ancestry_len: 3, + claim_queue_support: true, + }; collations.fetching_from = None; @@ -173,8 +182,11 @@ fn collation_fetching_fallback_works() { let claim_queue = vec![para_a]; let mut collations = Collations::new(&claim_queue); - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let relay_parent_mode = ProspectiveParachainsMode::Enabled { + max_candidate_depth: 4, + allowed_ancestry_len: 3, + claim_queue_support: false, + }; collations.fetching_from = None; @@ -206,6 +218,7 @@ fn collation_fetching_fallback_works() { collator_id_a.clone(), ); + // Collations will be fetched in the order they were added collations.add_to_waiting_queue(collation_a1.clone()); collations.add_to_waiting_queue(collation_a2.clone()); @@ -246,8 +259,11 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { let claim_queue = vec![para_a, para_b, para_a, para_b]; let mut collations = Collations::new(&claim_queue); - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 5, allowed_ancestry_len: 4 }; + let relay_parent_mode = ProspectiveParachainsMode::Enabled { + max_candidate_depth: 5, + allowed_ancestry_len: 4, + claim_queue_support: true, + }; collations.fetching_from = None; diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 70d189334a95..91208176747c 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -132,6 +132,23 @@ pub(super) async fn update_view( } ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, + RuntimeApiRequest::Version(tx), + )) => { + match test_state.claim_queue { + Some(_) => { + let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); + }, + None => { + let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT - 1)); + } + } + } + ); + assert_assign_incoming( virtual_overseer, test_state, @@ -1660,7 +1677,7 @@ fn collation_fetches_without_claimqueue() { ); // in fallback mode up to `max_candidate_depth` collations are accepted - for i in 0..test_state.async_backing_params.max_candidate_depth { + for i in 0..test_state.async_backing_params.max_candidate_depth + 1 { submit_second_and_assert( &mut virtual_overseer, keystore.clone(), diff --git a/polkadot/node/subsystem-util/src/runtime/mod.rs b/polkadot/node/subsystem-util/src/runtime/mod.rs index 2c9ec8db3778..7902848b4937 100644 --- a/polkadot/node/subsystem-util/src/runtime/mod.rs +++ b/polkadot/node/subsystem-util/src/runtime/mod.rs @@ -38,11 +38,12 @@ use polkadot_primitives::{ }; use crate::{ - request_async_backing_params, request_availability_cores, request_candidate_events, - request_from_runtime, request_key_ownership_proof, request_on_chain_votes, - request_session_executor_params, request_session_index_for_child, request_session_info, - request_submit_report_dispute_lost, request_unapplied_slashes, request_validation_code_by_hash, - request_validator_groups, vstaging::get_disabled_validators_with_fallback, + has_required_runtime, request_async_backing_params, request_availability_cores, + request_candidate_events, request_from_runtime, request_key_ownership_proof, + request_on_chain_votes, request_session_executor_params, request_session_index_for_child, + request_session_info, request_submit_report_dispute_lost, request_unapplied_slashes, + request_validation_code_by_hash, request_validator_groups, + vstaging::get_disabled_validators_with_fallback, }; /// Errors that can happen on runtime fetches. @@ -481,6 +482,8 @@ pub enum ProspectiveParachainsMode { /// How many ancestors of a relay parent are allowed to build candidates on top /// of. allowed_ancestry_len: usize, + /// Whether or not the runtime supports claim queue. + claim_queue_support: bool, }, } @@ -489,6 +492,14 @@ impl ProspectiveParachainsMode { pub fn is_enabled(&self) -> bool { matches!(self, ProspectiveParachainsMode::Enabled { .. }) } + + /// Returns true if ProspectiveParachainsMode is enabled and has got claim queue support + pub fn has_claim_queue_support(&self) -> bool { + match self { + ProspectiveParachainsMode::Enabled { claim_queue_support, .. } => *claim_queue_support, + ProspectiveParachainsMode::Disabled => false, + } + } } /// Requests prospective parachains mode for a given relay parent based on @@ -515,9 +526,16 @@ where Ok(ProspectiveParachainsMode::Disabled) } else { let AsyncBackingParams { max_candidate_depth, allowed_ancestry_len } = result?; + let claim_queue_support = has_required_runtime( + sender, + relay_parent, + RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT, + ) + .await; Ok(ProspectiveParachainsMode::Enabled { max_candidate_depth: max_candidate_depth as _, allowed_ancestry_len: allowed_ancestry_len as _, + claim_queue_support, }) } } From a10c86d5eca005ffcfbdfe4c468c84c61bf616a4 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 3 Jul 2024 14:49:32 +0300 Subject: [PATCH 17/55] Separate `claim_queue_support` from `ProspectiveParachainsMode` --- polkadot/node/core/backing/src/lib.rs | 5 +- .../tests/prospective_parachains.rs | 20 ------- .../src/validator_side/collation.rs | 14 ++--- .../src/validator_side/mod.rs | 20 +++++-- .../src/validator_side/tests/collation.rs | 54 ++++++++----------- .../src/validator_side/tests/mod.rs | 9 ++++ .../node/subsystem-util/src/runtime/mod.rs | 28 ++-------- 7 files changed, 63 insertions(+), 87 deletions(-) diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 1bda81c5197e..9498e0d657e0 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -281,7 +281,10 @@ impl From<&ActiveLeafState> for ProspectiveParachainsMode { ActiveLeafState::ProspectiveParachainsEnabled { max_candidate_depth, allowed_ancestry_len, - } => ProspectiveParachainsMode::Enabled { max_candidate_depth, allowed_ancestry_len }, + } => ProspectiveParachainsMode::Enabled { + max_candidate_depth, + allowed_ancestry_len, + }, } } } diff --git a/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs index 9e68247d5b39..ea8fdb0e04fb 100644 --- a/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/collator_side/tests/prospective_parachains.rs @@ -58,16 +58,6 @@ async fn update_view( } ); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::Version(tx), - )) => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); - } - ); - let min_number = leaf_number.saturating_sub(ASYNC_BACKING_PARAMETERS.allowed_ancestry_len); let ancestry_len = leaf_number + 1 - min_number; @@ -106,16 +96,6 @@ async fn update_view( } ); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::Version(tx), - )) => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); - } - ); - assert_matches!( overseer_recv_with_timeout(virtual_overseer, Duration::from_millis(50)).await.unwrap(), AllMessages::RuntimeApi( diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 6c55c92681eb..6ec18a2946f3 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -245,6 +245,8 @@ pub struct Collations { // from `GroupAssignments` which contains either the claim queue for the core or the `ParaId` // of the parachain assigned to the core. claims_per_para: BTreeMap, + // Whether the runtime supports claim queue runtime api. + has_claim_queue: bool, } impl Collations { @@ -255,7 +257,7 @@ impl Collations { /// doesn't support claim queue `GroupAssignments` contains only one entry - the `ParaId` of the /// parachain assigned to the core. This way we can handle both cases with a single /// implementation and avoid code duplication. - pub(super) fn new(group_assignments: &Vec) -> Self { + pub(super) fn new(group_assignments: &Vec, has_claim_queue: bool) -> Self { let mut claims_per_para = BTreeMap::new(); for para_id in group_assignments { *claims_per_para.entry(*para_id).or_default() += 1; @@ -268,6 +270,7 @@ impl Collations { seconded_count: 0, fetched_per_para: Default::default(), claims_per_para, + has_claim_queue, } } @@ -342,15 +345,12 @@ impl Collations { ) -> bool { match relay_parent_mode { ProspectiveParachainsMode::Disabled => return self.seconded_count >= 1, - ProspectiveParachainsMode::Enabled { - max_candidate_depth, - allowed_ancestry_len: _, - claim_queue_support, - } if !claim_queue_support => self.seconded_count >= max_candidate_depth + 1, + ProspectiveParachainsMode::Enabled { max_candidate_depth, allowed_ancestry_len: _ } + if !self.has_claim_queue => + self.seconded_count >= max_candidate_depth + 1, ProspectiveParachainsMode::Enabled { max_candidate_depth: _, allowed_ancestry_len: _, - claim_queue_support: _, } => { // Successful fetches + pending fetches < claim queue entries for `para_id` let respected_per_para_limit = diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 437825033dc8..fc57f7795cb5 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -43,12 +43,13 @@ use polkadot_node_subsystem::{ messages::{ CanSecondRequest, CandidateBackingMessage, CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, NetworkBridgeTxMessage, ParentHeadData, ProspectiveParachainsMessage, - ProspectiveValidationDataRequest, + ProspectiveValidationDataRequest, RuntimeApiRequest, }, overseer, CollatorProtocolSenderTrait, FromOrchestra, OverseerSignal, PerLeafSpan, }; use polkadot_node_subsystem_util::{ backing_implicit_view::View as ImplicitView, + has_required_runtime, reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL}, runtime::{prospective_parachains_mode, ProspectiveParachainsMode}, vstaging::fetch_claim_queue, @@ -369,8 +370,12 @@ struct PerRelayParent { } impl PerRelayParent { - fn new(mode: ProspectiveParachainsMode, assignments: GroupAssignments) -> Self { - let collations = Collations::new(&assignments.current); + fn new( + mode: ProspectiveParachainsMode, + assignments: GroupAssignments, + has_claim_queue: bool, + ) -> Self { + let collations = Collations::new(&assignments.current, has_claim_queue); Self { prospective_parachains_mode: mode, assignment: assignments, collations } } } @@ -1262,6 +1267,9 @@ where for leaf in added { let mode = prospective_parachains_mode(sender, *leaf).await?; + let has_claim_queue_support = + has_required_runtime(sender, *leaf, RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT) + .await; if let Some(span) = view.span_per_head().get(leaf).cloned() { let per_leaf_span = PerLeafSpan::new(span, "validator-side"); @@ -1272,7 +1280,9 @@ where assign_incoming(sender, &mut state.current_assignments, keystore, *leaf, mode).await?; state.active_leaves.insert(*leaf, mode); - state.per_relay_parent.insert(*leaf, PerRelayParent::new(mode, assignments)); + state + .per_relay_parent + .insert(*leaf, PerRelayParent::new(mode, assignments, has_claim_queue_support)); if mode.is_enabled() { state @@ -1297,7 +1307,7 @@ where ) .await?; - entry.insert(PerRelayParent::new(mode, assignments)); + entry.insert(PerRelayParent::new(mode, assignments, has_claim_queue_support)); } } } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 11b12a52efdf..b0a252d9ef0b 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -13,16 +13,14 @@ fn cant_add_more_than_claim_queue() { let para_a = ParaId::from(1); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; - let relay_parent_mode = ProspectiveParachainsMode::Enabled { - max_candidate_depth: 4, - allowed_ancestry_len: 3, - claim_queue_support: true, - }; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let claim_queue_support = true; - let mut collations = Collations::new(&assignments); + let mut collations = Collations::new(&assignments, claim_queue_support); // first collation for `para_a` is in the limit - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); + assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 0,)); collations.note_fetched(para_a); // and `para_b` is not affected assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); @@ -54,13 +52,11 @@ fn pending_fetches_are_counted() { let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; - let relay_parent_mode = ProspectiveParachainsMode::Enabled { - max_candidate_depth: 4, - allowed_ancestry_len: 3, - claim_queue_support: true, - }; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let claim_queue_support = true; - let mut collations = Collations::new(&assignments); + let mut collations = Collations::new(&assignments, claim_queue_support); collations.fetching_from = Some((collator_id_a, None)); // first collation for `para_a` is in the limit @@ -84,12 +80,11 @@ fn collation_fetching_respects_claim_queue() { let peer_b = PeerId::random(); let claim_queue = vec![para_a, para_b, para_a]; - let mut collations = Collations::new(&claim_queue); - let relay_parent_mode = ProspectiveParachainsMode::Enabled { - max_candidate_depth: 4, - allowed_ancestry_len: 3, - claim_queue_support: true, - }; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let claim_queue_support = true; + + let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; @@ -181,12 +176,11 @@ fn collation_fetching_fallback_works() { let peer_a = PeerId::random(); let claim_queue = vec![para_a]; - let mut collations = Collations::new(&claim_queue); - let relay_parent_mode = ProspectiveParachainsMode::Enabled { - max_candidate_depth: 4, - allowed_ancestry_len: 3, - claim_queue_support: false, - }; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let claim_queue_support = false; + + let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; @@ -258,13 +252,11 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { let peer_b = PeerId::random(); let claim_queue = vec![para_a, para_b, para_a, para_b]; - let mut collations = Collations::new(&claim_queue); - let relay_parent_mode = ProspectiveParachainsMode::Enabled { - max_candidate_depth: 5, - allowed_ancestry_len: 4, - claim_queue_support: true, - }; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 5, allowed_ancestry_len: 4 }; + let claim_queue_support = true; + let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; let relay_parent = Hash::repeat_byte(0x01); diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 7243af9fe4ca..162bde1a41e4 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -564,6 +564,15 @@ async fn assert_async_backing_params_request(virtual_overseer: &mut VirtualOvers tx.send(Err(ASYNC_BACKING_DISABLED_ERROR)).unwrap(); } ); + assert_matches!( + overseer_recv(virtual_overseer).await, + AllMessages::RuntimeApi(RuntimeApiMessage::Request( + _, + RuntimeApiRequest::Version(tx), + )) => { + let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); + } + ); } // As we receive a relevant advertisement act on it and issue a collation request. diff --git a/polkadot/node/subsystem-util/src/runtime/mod.rs b/polkadot/node/subsystem-util/src/runtime/mod.rs index 7902848b4937..2c9ec8db3778 100644 --- a/polkadot/node/subsystem-util/src/runtime/mod.rs +++ b/polkadot/node/subsystem-util/src/runtime/mod.rs @@ -38,12 +38,11 @@ use polkadot_primitives::{ }; use crate::{ - has_required_runtime, request_async_backing_params, request_availability_cores, - request_candidate_events, request_from_runtime, request_key_ownership_proof, - request_on_chain_votes, request_session_executor_params, request_session_index_for_child, - request_session_info, request_submit_report_dispute_lost, request_unapplied_slashes, - request_validation_code_by_hash, request_validator_groups, - vstaging::get_disabled_validators_with_fallback, + request_async_backing_params, request_availability_cores, request_candidate_events, + request_from_runtime, request_key_ownership_proof, request_on_chain_votes, + request_session_executor_params, request_session_index_for_child, request_session_info, + request_submit_report_dispute_lost, request_unapplied_slashes, request_validation_code_by_hash, + request_validator_groups, vstaging::get_disabled_validators_with_fallback, }; /// Errors that can happen on runtime fetches. @@ -482,8 +481,6 @@ pub enum ProspectiveParachainsMode { /// How many ancestors of a relay parent are allowed to build candidates on top /// of. allowed_ancestry_len: usize, - /// Whether or not the runtime supports claim queue. - claim_queue_support: bool, }, } @@ -492,14 +489,6 @@ impl ProspectiveParachainsMode { pub fn is_enabled(&self) -> bool { matches!(self, ProspectiveParachainsMode::Enabled { .. }) } - - /// Returns true if ProspectiveParachainsMode is enabled and has got claim queue support - pub fn has_claim_queue_support(&self) -> bool { - match self { - ProspectiveParachainsMode::Enabled { claim_queue_support, .. } => *claim_queue_support, - ProspectiveParachainsMode::Disabled => false, - } - } } /// Requests prospective parachains mode for a given relay parent based on @@ -526,16 +515,9 @@ where Ok(ProspectiveParachainsMode::Disabled) } else { let AsyncBackingParams { max_candidate_depth, allowed_ancestry_len } = result?; - let claim_queue_support = has_required_runtime( - sender, - relay_parent, - RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT, - ) - .await; Ok(ProspectiveParachainsMode::Enabled { max_candidate_depth: max_candidate_depth as _, allowed_ancestry_len: allowed_ancestry_len as _, - claim_queue_support, }) } } From b39858aaafb6fed18767eb8567e5a26b03246013 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 3 Jul 2024 15:01:00 +0300 Subject: [PATCH 18/55] Fix comments and add logs --- .../src/validator_side/collation.rs | 47 +++++++++++++------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 6ec18a2946f3..dc803da7506d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -242,8 +242,8 @@ pub struct Collations { /// What collations were fetched so far for this relay parent. fetched_per_para: BTreeMap, // Claims per `ParaId` for the assigned core at the relay parent. This information is obtained - // from `GroupAssignments` which contains either the claim queue for the core or the `ParaId` - // of the parachain assigned to the core. + // from `GroupAssignments` which contains either the claim queue (if runtime supports it) for + // the core or the `ParaId` of the parachain assigned to the core. claims_per_para: BTreeMap, // Whether the runtime supports claim queue runtime api. has_claim_queue: bool, @@ -255,8 +255,7 @@ impl Collations { /// uses `GroupAssignments`. If the runtime supports claim queue `GroupAssignments` contains the /// claim queue for the core assigned to the (backing group of the) validator. If the runtime /// doesn't support claim queue `GroupAssignments` contains only one entry - the `ParaId` of the - /// parachain assigned to the core. This way we can handle both cases with a single - /// implementation and avoid code duplication. + /// parachain assigned to the core. pub(super) fn new(group_assignments: &Vec, has_claim_queue: bool) -> Self { let mut claims_per_para = BTreeMap::new(); for para_id in group_assignments { @@ -332,7 +331,7 @@ impl Collations { /// this case there is a limit of 1 collation per relay parent. /// /// If prospective parachains mode is enabled but claim queue is not supported then up to - /// `max_candidate_depth + 1` seconded collations are supported. In theory in this case if two + /// `max_candidate_depth + 1` seconded collations are accepted. In theory in this case if two /// parachains are sharing a core no fairness is guaranteed between them and the faster one can /// starve the slower one by exhausting the limit with its own advertisements. In practice this /// should not happen because core sharing implies core time support which implies the claim @@ -344,10 +343,29 @@ impl Collations { num_pending_fetches: usize, ) -> bool { match relay_parent_mode { - ProspectiveParachainsMode::Disabled => return self.seconded_count >= 1, + ProspectiveParachainsMode::Disabled => { + gum::trace!( + target: LOG_TARGET, + ?para_id, + seconded_count=self.seconded_count, + "is_collations_limit_reached - ProspectiveParachainsMode::Disabled" + ); + + self.seconded_count >= 1 + }, ProspectiveParachainsMode::Enabled { max_candidate_depth, allowed_ancestry_len: _ } if !self.has_claim_queue => - self.seconded_count >= max_candidate_depth + 1, + { + gum::trace!( + target: LOG_TARGET, + ?para_id, + seconded_count=self.seconded_count, + max_candidate_depth, + "is_collations_limit_reached - ProspectiveParachainsMode::Enabled without claim queue support" + ); + + self.seconded_count >= max_candidate_depth + 1 + }, ProspectiveParachainsMode::Enabled { max_candidate_depth: _, allowed_ancestry_len: _, @@ -365,7 +383,7 @@ impl Collations { fetched_per_para=?self.fetched_per_para, ?num_pending_fetches, ?respected_per_para_limit, - "is_collations_limit_reached" + "is_collations_limit_reached - ProspectiveParachainsMode::Enabled with claim queue support" ); !respected_per_para_limit @@ -375,7 +393,7 @@ impl Collations { /// Adds a new collation to the waiting queue for the relay parent. This function doesn't /// perform any limits check. The caller (`enqueue_collation`) should assure that the collation - /// can be enqueued. + /// limit is respected. pub(super) fn add_to_waiting_queue(&mut self, collation: (PendingCollation, CollatorId)) { self.waiting_queue.entry(collation.0.para_id).or_default().push_back(collation); } @@ -383,15 +401,14 @@ impl Collations { /// Picks a collation to fetch from the waiting queue. /// When fetching collations we need to ensure that each parachain has got a fair core time /// share depending on its assignments in the claim queue. This means that the number of - /// collations fetched per parachain should ideally be equal to (but not bigger than) the number - /// of claims for the particular parachain in the claim queue. + /// collations fetched per parachain should ideally be equal to the number of claims for the + /// particular parachain in the claim queue. /// /// To achieve this each parachain with at an entry in the `waiting_queue` has got a score /// calculated by dividing the number of fetched collations by the number of entries in the - /// claim queue. Lower the score means higher fetching priority. Note that if a parachain hasn't - /// got anything fetched at this relay parent it will have score 0 which means highest priority. - /// - /// If two parachains has got the same score the one which is earlier in the claim queue will be + /// claim queue. Lower score means higher fetching priority. Note that if a parachain hasn't got + /// anything fetched at this relay parent it will have score 0 which means highest priority. If + /// two parachains has got the same score the one which is earlier in the claim queue will be /// picked. /// /// If claim queue is not supported then `group_assignment` should contain just one element and From b30f3407a508aa67c481066c6e6fde262780924f Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 3 Jul 2024 15:11:58 +0300 Subject: [PATCH 19/55] Update test: `collation_fetching_prefer_entries_earlier_in_claim_queue` --- .../src/validator_side/tests/collation.rs | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index b0a252d9ef0b..548e3c90a413 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -251,9 +251,13 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); let peer_b = PeerId::random(); - let claim_queue = vec![para_a, para_b, para_a, para_b]; + let para_c = ParaId::from(3); + let collator_id_c = CollatorId::from(sr25519::Public::from_raw([30u8; 32])); + let peer_c = PeerId::random(); + + let claim_queue = vec![para_a, para_b, para_a, para_b, para_c, para_c]; let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 5, allowed_ancestry_len: 4 }; + ProspectiveParachainsMode::Enabled { max_candidate_depth: 6, allowed_ancestry_len: 5 }; let claim_queue_support = true; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -313,7 +317,35 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collator_id_b.clone(), ); - // Despite the order here the fetches should be a1, b1, a2, b2 + let collation_c1 = ( + PendingCollation::new( + relay_parent, + para_c, + &peer_c, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(5)), + parent_head_data_hash: Hash::repeat_byte(5), + }), + ), + collator_id_c.clone(), + ); + + let collation_c2 = ( + PendingCollation::new( + relay_parent, + para_c, + &peer_c, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(6)), + parent_head_data_hash: Hash::repeat_byte(6), + }), + ), + collator_id_c.clone(), + ); + + // Despite the order here the fetches should be a1, b1, c1, a2, b2, c2 + collations.add_to_waiting_queue(collation_c1.clone()); + collations.add_to_waiting_queue(collation_c2.clone()); collations.add_to_waiting_queue(collation_b1.clone()); collations.add_to_waiting_queue(collation_b2.clone()); collations.add_to_waiting_queue(collation_a1.clone()); @@ -341,6 +373,17 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { ); collations.note_fetched(collation_b1.0.para_id); + assert_eq!( + Some(collation_c1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_c1.0.para_id); + assert_eq!( Some(collation_a2.clone()), collations.get_next_collation_to_fetch( @@ -362,4 +405,15 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { ) ); collations.note_fetched(collation_b2.0.para_id); + + assert_eq!( + Some(collation_c2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_c2.0.para_id); } From c0f18b99c43537463fc18abbfe894bee27146084 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 3 Jul 2024 15:27:13 +0300 Subject: [PATCH 20/55] Fix `pick_a_collation_to_fetch` and more tests --- .../src/validator_side/collation.rs | 3 +- .../src/validator_side/tests/collation.rs | 458 +++++++++++++++++- 2 files changed, 459 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index dc803da7506d..d2db184d5298 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -448,9 +448,10 @@ impl Collations { match lowest_score { Some((score, _)) if para_score < score => lowest_score = Some((para_score, vec![(para_id, collations)])), - Some((_, ref mut paras)) => { + Some((score, ref mut paras)) if score == para_score => { paras.push((para_id, collations)); }, + Some(_) => continue, None => lowest_score = Some((para_score, vec![(para_id, collations)])), } } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 548e3c90a413..c00451742830 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -257,7 +257,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { let claim_queue = vec![para_a, para_b, para_a, para_b, para_c, para_c]; let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 6, allowed_ancestry_len: 5 }; + ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -417,3 +417,459 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { ); collations.note_fetched(collation_c2.0.para_id); } + +#[test] +fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { + sp_tracing::init_for_tests(); + + let para_a = ParaId::from(1); + let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); + let peer_a = PeerId::random(); + + let para_b = ParaId::from(2); + let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); + let peer_b = PeerId::random(); + + let para_c = ParaId::from(3); + let collator_id_c = CollatorId::from(sr25519::Public::from_raw([30u8; 32])); + let peer_c = PeerId::random(); + + let claim_queue = vec![para_a, para_b, para_a, para_b, para_a, para_b, para_c, para_c]; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 9, allowed_ancestry_len: 8 }; + let claim_queue_support = true; + + let mut collations = Collations::new(&claim_queue, claim_queue_support); + collations.fetching_from = None; + + let relay_parent = Hash::repeat_byte(0x01); + + let collation_a1 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(1)), + parent_head_data_hash: Hash::repeat_byte(1), + }), + ), + collator_id_a.clone(), + ); + + let collation_a2 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(2)), + parent_head_data_hash: Hash::repeat_byte(2), + }), + ), + collator_id_a.clone(), + ); + + let collation_a3 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(3)), + parent_head_data_hash: Hash::repeat_byte(3), + }), + ), + collator_id_a.clone(), + ); + + let collation_b1 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(4)), + parent_head_data_hash: Hash::repeat_byte(4), + }), + ), + collator_id_b.clone(), + ); + + let collation_b2 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(5)), + parent_head_data_hash: Hash::repeat_byte(5), + }), + ), + collator_id_b.clone(), + ); + + let collation_b3 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(6)), + parent_head_data_hash: Hash::repeat_byte(6), + }), + ), + collator_id_b.clone(), + ); + + let collation_c1 = ( + PendingCollation::new( + relay_parent, + para_c, + &peer_c, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(7)), + parent_head_data_hash: Hash::repeat_byte(7), + }), + ), + collator_id_c.clone(), + ); + + let collation_c2 = ( + PendingCollation::new( + relay_parent, + para_c, + &peer_c, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(8)), + parent_head_data_hash: Hash::repeat_byte(8), + }), + ), + collator_id_c.clone(), + ); + + collations.add_to_waiting_queue(collation_c1.clone()); + collations.add_to_waiting_queue(collation_c2.clone()); + collations.add_to_waiting_queue(collation_b1.clone()); + collations.add_to_waiting_queue(collation_b2.clone()); + collations.add_to_waiting_queue(collation_b3.clone()); + collations.add_to_waiting_queue(collation_a1.clone()); + collations.add_to_waiting_queue(collation_a2.clone()); + collations.add_to_waiting_queue(collation_a3.clone()); + + assert_eq!( + Some(collation_a1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a1.0.para_id); + + assert_eq!( + Some(collation_b1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b1.0.para_id); + + assert_eq!( + Some(collation_c1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_c1.0.para_id); + + assert_eq!( + Some(collation_a2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a2.0.para_id); + + assert_eq!( + Some(collation_b2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b2.0.para_id); + + assert_eq!( + Some(collation_c2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_c2.0.para_id); + + assert_eq!( + Some(collation_a3.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a3.0.para_id); + + assert_eq!( + Some(collation_b3.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b3.0.para_id); +} + +#[test] +fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { + sp_tracing::init_for_tests(); + + let para_a = ParaId::from(1); + let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); + let peer_a = PeerId::random(); + + let para_b = ParaId::from(2); + let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); + let peer_b = PeerId::random(); + + let para_c = ParaId::from(3); + let collator_id_c = CollatorId::from(sr25519::Public::from_raw([30u8; 32])); + let peer_c = PeerId::random(); + + let claim_queue = vec![para_a, para_a, para_a, para_a, para_b, para_b, para_c, para_c]; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 9, allowed_ancestry_len: 8 }; + let claim_queue_support = true; + + let mut collations = Collations::new(&claim_queue, claim_queue_support); + collations.fetching_from = None; + + let relay_parent = Hash::repeat_byte(0x01); + + let collation_a1 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(1)), + parent_head_data_hash: Hash::repeat_byte(1), + }), + ), + collator_id_a.clone(), + ); + + let collation_a2 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(2)), + parent_head_data_hash: Hash::repeat_byte(2), + }), + ), + collator_id_a.clone(), + ); + + let collation_a3 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(3)), + parent_head_data_hash: Hash::repeat_byte(3), + }), + ), + collator_id_a.clone(), + ); + + let collation_a4 = ( + PendingCollation::new( + relay_parent, + para_a, + &peer_a, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(4)), + parent_head_data_hash: Hash::repeat_byte(4), + }), + ), + collator_id_a.clone(), + ); + + let collation_b1 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(5)), + parent_head_data_hash: Hash::repeat_byte(5), + }), + ), + collator_id_b.clone(), + ); + + let collation_b2 = ( + PendingCollation::new( + relay_parent, + para_b, + &peer_b, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(6)), + parent_head_data_hash: Hash::repeat_byte(6), + }), + ), + collator_id_b.clone(), + ); + + let collation_c1 = ( + PendingCollation::new( + relay_parent, + para_c, + &peer_c, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(7)), + parent_head_data_hash: Hash::repeat_byte(7), + }), + ), + collator_id_c.clone(), + ); + + let collation_c2 = ( + PendingCollation::new( + relay_parent, + para_c, + &peer_c, + Some(ProspectiveCandidate { + candidate_hash: CandidateHash(Hash::repeat_byte(8)), + parent_head_data_hash: Hash::repeat_byte(8), + }), + ), + collator_id_c.clone(), + ); + + collations.add_to_waiting_queue(collation_c1.clone()); + collations.add_to_waiting_queue(collation_c2.clone()); + collations.add_to_waiting_queue(collation_b1.clone()); + collations.add_to_waiting_queue(collation_b2.clone()); + collations.add_to_waiting_queue(collation_a1.clone()); + collations.add_to_waiting_queue(collation_a2.clone()); + collations.add_to_waiting_queue(collation_a3.clone()); + collations.add_to_waiting_queue(collation_a4.clone()); + + assert_eq!( + Some(collation_a1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a1.0.para_id); + + assert_eq!( + Some(collation_b1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b1.0.para_id); + + assert_eq!( + Some(collation_c1.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_c1.0.para_id); + + assert_eq!( + Some(collation_a2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a2.0.para_id); + + assert_eq!( + Some(collation_a3.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a3.0.para_id); + + assert_eq!( + Some(collation_b2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_b2.0.para_id); + + assert_eq!( + Some(collation_c2.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_c2.0.para_id); + + assert_eq!( + Some(collation_a4.clone()), + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode.clone(), + &claim_queue, + ) + ); + collations.note_fetched(collation_a4.0.para_id); +} From fba7ca69abf830d559b6f84805fd67d85ced70db Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 4 Jul 2024 09:54:34 +0300 Subject: [PATCH 21/55] Fix `pick_a_collation_to_fetch` - iter 1 --- .../src/validator_side/collation.rs | 84 +++--- .../src/validator_side/mod.rs | 36 ++- .../src/validator_side/tests/collation.rs | 270 +++++------------- .../src/validator_side/tests/mod.rs | 2 +- 4 files changed, 143 insertions(+), 249 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index d2db184d5298..8c3dba5f959f 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -294,6 +294,7 @@ impl Collations { finished_one: &(CollatorId, Option), relay_parent_mode: ProspectiveParachainsMode, group_assignments: &Vec, + pending_fetches: &BTreeMap, ) -> Option<(PendingCollation, CollatorId)> { // If finished one does not match waiting_collation, then we already dequeued another fetch // to replace it. @@ -318,7 +319,8 @@ impl Collations { // `Waiting` so that we can fetch more collations. If async backing is disabled we can't // fetch more than one collation per relay parent so `None` is returned. CollationStatus::Seconded => None, - CollationStatus::Waiting => self.pick_a_collation_to_fetch(&group_assignments), + CollationStatus::Waiting => + self.pick_a_collation_to_fetch(&group_assignments, pending_fetches), CollationStatus::WaitingOnValidation | CollationStatus::Fetching => unreachable!("We have reset the status above!"), } @@ -417,6 +419,7 @@ impl Collations { fn pick_a_collation_to_fetch( &mut self, group_assignments: &Vec, + pending_fetches: &BTreeMap, ) -> Option<(PendingCollation, CollatorId)> { gum::trace!( target: LOG_TARGET, @@ -427,51 +430,56 @@ impl Collations { "Pick a collation to fetch." ); - // Find the parachain(s) with the lowest score. - let mut lowest_score = None; - for (para_id, collations) in &mut self.waiting_queue { - let para_score = self - .fetched_per_para - .get(para_id) - .copied() - .map(|v| { - (v as f64) / - (self.claims_per_para.get(para_id).copied().unwrap_or_default() as f64) - }) - .unwrap_or_default(); - - // skip empty queues - if collations.is_empty() { - continue + if !self.has_claim_queue { + if let Some(assigned_para_id) = group_assignments.first() { + return self + .waiting_queue + .get_mut(assigned_para_id) + .map(|collations| collations.pop_front()) + .flatten() + } else { + unreachable!("Group assignments should contain at least one element.") } + } - match lowest_score { - Some((score, _)) if para_score < score => - lowest_score = Some((para_score, vec![(para_id, collations)])), - Some((score, ref mut paras)) if score == para_score => { - paras.push((para_id, collations)); - }, - Some(_) => continue, - None => lowest_score = Some((para_score, vec![(para_id, collations)])), + let mut claim_queue_state = Vec::with_capacity(group_assignments.len()); + let mut fetched_per_para = self.fetched_per_para.clone(); + let mut pending_fetches = pending_fetches.clone(); + + for assignment in group_assignments { + if let Some(fetched) = fetched_per_para.get_mut(assignment) { + if *fetched > 0 { + claim_queue_state.push((true, assignment)); + *fetched -= 1; + continue; + } } + + if let Some(pending_fetches) = pending_fetches.get_mut(assignment) { + if *pending_fetches > 0 { + claim_queue_state.push((true, assignment)); + *pending_fetches -= 1; + continue; + } + } + + claim_queue_state.push((false, assignment)); } - if let Some((_, mut lowest_score)) = lowest_score { - for claim in group_assignments { - if let Some((_, collations)) = lowest_score.iter_mut().find(|(id, _)| *id == claim) - { - match collations.pop_front() { - Some(collation) => return Some(collation), - None => { - unreachable!("Collation can't be empty because empty ones are skipped at the beginning of the loop.") - }, - } + for (fulfilled, assignment) in &mut claim_queue_state { + if *fulfilled { + continue + } + + if let Some(collations) = self.waiting_queue.get_mut(assignment) { + if let Some(collation) = collations.pop_front() { + *fulfilled = true; + return Some(collation) } } - unreachable!("All entries in waiting_queue should also be in claim queue") - } else { - None } + + None } } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index fc57f7795cb5..63062900a0c3 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -19,7 +19,7 @@ use futures::{ }; use futures_timer::Delay; use std::{ - collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, + collections::{hash_map::Entry, BTreeMap, HashMap, HashSet, VecDeque}, future::Future, time::{Duration, Instant}, }; @@ -1738,13 +1738,16 @@ async fn dequeue_next_collation_and_fetch( // The collator we tried to fetch from last, optionally which candidate. previous_fetch: (CollatorId, Option), ) { - while let Some((next, id)) = state.per_relay_parent.get_mut(&relay_parent).and_then(|state| { - state.collations.get_next_collation_to_fetch( - &previous_fetch, - state.prospective_parachains_mode, - &state.assignment.current, - ) - }) { + let pending_collations = pending_collations_per_para_at_relay_parent(state, relay_parent); + while let Some((next, id)) = + state.per_relay_parent.get_mut(&relay_parent).and_then(|rp_state| { + rp_state.collations.get_next_collation_to_fetch( + &previous_fetch, + rp_state.prospective_parachains_mode, + &rp_state.assignment.current, + &pending_collations, + ) + }) { gum::debug!( target: LOG_TARGET, ?relay_parent, @@ -2117,7 +2120,7 @@ async fn handle_collation_fetch_response( result } -// Returns the number of pending fetches for `ParaId` at a specific relay parent. +// Returns the number of pending fetches for `ParaId` at the specified relay parent. fn num_pending_collations_for_para_at_relay_parent( state: &State, para_id: ParaId, @@ -2131,3 +2134,18 @@ fn num_pending_collations_for_para_at_relay_parent( }) .count() } + +// Returns the number of pending fetches for each `ParaId` at the specified relay parent. +fn pending_collations_per_para_at_relay_parent( + state: &State, + relay_parent: Hash, +) -> BTreeMap { + state + .collation_requests_cancel_handles + .iter() + .filter(|(col, _)| col.relay_parent == relay_parent) + .fold(BTreeMap::new(), |mut res, (col, _)| { + *res.entry(col.para_id).or_default() += 1; + res + }) +} diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index c00451742830..143cdf8fed36 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use polkadot_node_subsystem_util::runtime::ProspectiveParachainsMode; use polkadot_primitives::{CandidateHash, CollatorId, Hash, Id as ParaId}; @@ -83,6 +85,7 @@ fn collation_fetching_respects_claim_queue() { let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; let claim_queue_support = true; + let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -140,6 +143,7 @@ fn collation_fetching_respects_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending ) ); collations.note_fetched(collation_a1.0.para_id); @@ -151,6 +155,7 @@ fn collation_fetching_respects_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending ) ); collations.note_fetched(collation_b1.0.para_id); @@ -162,6 +167,7 @@ fn collation_fetching_respects_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending ) ); collations.note_fetched(collation_a2.0.para_id); @@ -179,6 +185,7 @@ fn collation_fetching_fallback_works() { let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; let claim_queue_support = false; + let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -223,6 +230,7 @@ fn collation_fetching_fallback_works() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_a1.0.para_id); @@ -234,6 +242,7 @@ fn collation_fetching_fallback_works() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_a2.0.para_id); @@ -259,6 +268,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; + let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; @@ -358,6 +368,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_a1.0.para_id); @@ -369,42 +380,46 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_b1.0.para_id); assert_eq!( - Some(collation_c1.clone()), + Some(collation_a2.clone()), collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); - collations.note_fetched(collation_c1.0.para_id); + collations.note_fetched(collation_a2.0.para_id); assert_eq!( - Some(collation_a2.clone()), + Some(collation_b2.clone()), collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); - collations.note_fetched(collation_a2.0.para_id); + collations.note_fetched(collation_b2.0.para_id); assert_eq!( - Some(collation_b2.clone()), + Some(collation_c1.clone()), collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); - collations.note_fetched(collation_b2.0.para_id); + collations.note_fetched(collation_c1.0.para_id); assert_eq!( Some(collation_c2.clone()), @@ -413,13 +428,14 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_c2.0.para_id); } #[test] -fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { +fn collation_fetching_fills_holes_in_claim_queue() { sp_tracing::init_for_tests(); let para_a = ParaId::from(1); @@ -434,10 +450,11 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { let collator_id_c = CollatorId::from(sr25519::Public::from_raw([30u8; 32])); let peer_c = PeerId::random(); - let claim_queue = vec![para_a, para_b, para_a, para_b, para_a, para_b, para_c, para_c]; + let claim_queue = vec![para_a, para_b, para_a, para_b, para_c, para_c]; let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 9, allowed_ancestry_len: 8 }; + ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; + let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; @@ -470,27 +487,14 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { collator_id_a.clone(), ); - let collation_a3 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(3)), - parent_head_data_hash: Hash::repeat_byte(3), - }), - ), - collator_id_a.clone(), - ); - let collation_b1 = ( PendingCollation::new( relay_parent, para_b, &peer_b, Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(4)), - parent_head_data_hash: Hash::repeat_byte(4), + candidate_hash: CandidateHash(Hash::repeat_byte(3)), + parent_head_data_hash: Hash::repeat_byte(3), }), ), collator_id_b.clone(), @@ -502,21 +506,8 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { para_b, &peer_b, Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(5)), - parent_head_data_hash: Hash::repeat_byte(5), - }), - ), - collator_id_b.clone(), - ); - - let collation_b3 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(6)), - parent_head_data_hash: Hash::repeat_byte(6), + candidate_hash: CandidateHash(Hash::repeat_byte(4)), + parent_head_data_hash: Hash::repeat_byte(4), }), ), collator_id_b.clone(), @@ -528,8 +519,8 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { para_c, &peer_c, Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(7)), - parent_head_data_hash: Hash::repeat_byte(7), + candidate_hash: CandidateHash(Hash::repeat_byte(5)), + parent_head_data_hash: Hash::repeat_byte(5), }), ), collator_id_c.clone(), @@ -541,21 +532,16 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { para_c, &peer_c, Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(8)), - parent_head_data_hash: Hash::repeat_byte(8), + candidate_hash: CandidateHash(Hash::repeat_byte(6)), + parent_head_data_hash: Hash::repeat_byte(6), }), ), collator_id_c.clone(), ); + // Despite the order here the fetches should be a1, b1, c1, a2, b2, c2 collations.add_to_waiting_queue(collation_c1.clone()); - collations.add_to_waiting_queue(collation_c2.clone()); - collations.add_to_waiting_queue(collation_b1.clone()); - collations.add_to_waiting_queue(collation_b2.clone()); - collations.add_to_waiting_queue(collation_b3.clone()); collations.add_to_waiting_queue(collation_a1.clone()); - collations.add_to_waiting_queue(collation_a2.clone()); - collations.add_to_waiting_queue(collation_a3.clone()); assert_eq!( Some(collation_a1.clone()), @@ -564,21 +550,11 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_a1.0.para_id); - assert_eq!( - Some(collation_b1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), - &claim_queue, - ) - ); - collations.note_fetched(collation_b1.0.para_id); - assert_eq!( Some(collation_c1.clone()), collations.get_next_collation_to_fetch( @@ -586,31 +562,25 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_c1.0.para_id); - assert_eq!( - Some(collation_a2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), - &claim_queue, - ) - ); - collations.note_fetched(collation_a2.0.para_id); + collations.add_to_waiting_queue(collation_c2.clone()); + collations.add_to_waiting_queue(collation_b1.clone()); assert_eq!( - Some(collation_b2.clone()), + Some(collation_b1.clone()), collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); - collations.note_fetched(collation_b2.0.para_id); + collations.note_fetched(collation_b1.0.para_id); assert_eq!( Some(collation_c2.clone()), @@ -619,35 +589,41 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue2() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); collations.note_fetched(collation_c2.0.para_id); + collations.add_to_waiting_queue(collation_b2.clone()); + collations.add_to_waiting_queue(collation_a2.clone()); + assert_eq!( - Some(collation_a3.clone()), + Some(collation_a2.clone()), collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); - collations.note_fetched(collation_a3.0.para_id); + collations.note_fetched(collation_a2.0.para_id); assert_eq!( - Some(collation_b3.clone()), + Some(collation_b2.clone()), collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &pending, ) ); - collations.note_fetched(collation_b3.0.para_id); + collations.note_fetched(collation_b2.0.para_id); } #[test] -fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { +fn collation_fetching_takes_in_account_pending_items() { sp_tracing::init_for_tests(); let para_a = ParaId::from(1); @@ -658,13 +634,9 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); let peer_b = PeerId::random(); - let para_c = ParaId::from(3); - let collator_id_c = CollatorId::from(sr25519::Public::from_raw([30u8; 32])); - let peer_c = PeerId::random(); - - let claim_queue = vec![para_a, para_a, para_a, para_a, para_b, para_b, para_c, para_c]; + let claim_queue = vec![para_a, para_b, para_a, para_b]; let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 9, allowed_ancestry_len: 8 }; + ProspectiveParachainsMode::Enabled { max_candidate_depth: 5, allowed_ancestry_len: 4 }; let claim_queue_support = true; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -698,40 +670,14 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { collator_id_a.clone(), ); - let collation_a3 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(3)), - parent_head_data_hash: Hash::repeat_byte(3), - }), - ), - collator_id_a.clone(), - ); - - let collation_a4 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(4)), - parent_head_data_hash: Hash::repeat_byte(4), - }), - ), - collator_id_a.clone(), - ); - let collation_b1 = ( PendingCollation::new( relay_parent, para_b, &peer_b, Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(5)), - parent_head_data_hash: Hash::repeat_byte(5), + candidate_hash: CandidateHash(Hash::repeat_byte(3)), + parent_head_data_hash: Hash::repeat_byte(3), }), ), collator_id_b.clone(), @@ -743,58 +689,16 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { para_b, &peer_b, Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(6)), - parent_head_data_hash: Hash::repeat_byte(6), + candidate_hash: CandidateHash(Hash::repeat_byte(4)), + parent_head_data_hash: Hash::repeat_byte(4), }), ), collator_id_b.clone(), ); - let collation_c1 = ( - PendingCollation::new( - relay_parent, - para_c, - &peer_c, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(7)), - parent_head_data_hash: Hash::repeat_byte(7), - }), - ), - collator_id_c.clone(), - ); - - let collation_c2 = ( - PendingCollation::new( - relay_parent, - para_c, - &peer_c, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(8)), - parent_head_data_hash: Hash::repeat_byte(8), - }), - ), - collator_id_c.clone(), - ); - - collations.add_to_waiting_queue(collation_c1.clone()); - collations.add_to_waiting_queue(collation_c2.clone()); - collations.add_to_waiting_queue(collation_b1.clone()); - collations.add_to_waiting_queue(collation_b2.clone()); - collations.add_to_waiting_queue(collation_a1.clone()); + // a1 will be pending, a2 and b1 will be in the queue; b1 should be fetched first collations.add_to_waiting_queue(collation_a2.clone()); - collations.add_to_waiting_queue(collation_a3.clone()); - collations.add_to_waiting_queue(collation_a4.clone()); - - assert_eq!( - Some(collation_a1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), - &claim_queue, - ) - ); - collations.note_fetched(collation_a1.0.para_id); + collations.add_to_waiting_queue(collation_b1.clone()); assert_eq!( Some(collation_b1.clone()), @@ -803,20 +707,13 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &BTreeMap::from([(para_a, 1)]), ) ); - collations.note_fetched(collation_b1.0.para_id); + collations.note_fetched(collation_a1.0.para_id); // a1 is no longer pending - assert_eq!( - Some(collation_c1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), - &claim_queue, - ) - ); - collations.note_fetched(collation_c1.0.para_id); + // a1 is fetched, b1 is pending, a2 and b2 are in the queue, a2 should be fetched next + collations.add_to_waiting_queue(collation_b2.clone()); assert_eq!( Some(collation_a2.clone()), @@ -825,21 +722,13 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &BTreeMap::from([(para_b, 1)]), ) ); + collations.note_fetched(collation_b1.0.para_id); collations.note_fetched(collation_a2.0.para_id); - assert_eq!( - Some(collation_a3.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), - &claim_queue, - ) - ); - collations.note_fetched(collation_a3.0.para_id); - + // and finally b2 should be fetched assert_eq!( Some(collation_b2.clone()), collations.get_next_collation_to_fetch( @@ -847,29 +736,8 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue3() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode.clone(), &claim_queue, + &BTreeMap::new(), ) ); collations.note_fetched(collation_b2.0.para_id); - - assert_eq!( - Some(collation_c2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), - &claim_queue, - ) - ); - collations.note_fetched(collation_c2.0.para_id); - - assert_eq!( - Some(collation_a4.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), - &claim_queue, - ) - ); - collations.note_fetched(collation_a4.0.para_id); } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 162bde1a41e4..5c49de54eb6f 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -570,7 +570,7 @@ async fn assert_async_backing_params_request(virtual_overseer: &mut VirtualOvers _, RuntimeApiRequest::Version(tx), )) => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); + let _ = tx.send(Ok(RuntimeApiRequest::ASYNC_BACKING_STATE_RUNTIME_REQUIREMENT - 1)); } ); } From d4f4ce2d52c99a347fe57a2c4217ee3f9aa9e58e Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 4 Jul 2024 11:02:39 +0300 Subject: [PATCH 22/55] Fix `pick_a_collation_to_fetch` - iter 2 --- .../src/validator_side/collation.rs | 96 +++++++++++-------- 1 file changed, 57 insertions(+), 39 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 8c3dba5f959f..88cfc86be657 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -245,8 +245,10 @@ pub struct Collations { // from `GroupAssignments` which contains either the claim queue (if runtime supports it) for // the core or the `ParaId` of the parachain assigned to the core. claims_per_para: BTreeMap, - // Whether the runtime supports claim queue runtime api. - has_claim_queue: bool, + // Represents the claim queue at the relay parent. The `bool` field indicates if a candidate + // was fetched for the `ParaId` at the position in question. In other words - if the claim is + // 'satisfied'. If the claim queue is not avaliable `claim_queue_state` will be `None`. + claim_queue_state: Option>, } impl Collations { @@ -258,10 +260,18 @@ impl Collations { /// parachain assigned to the core. pub(super) fn new(group_assignments: &Vec, has_claim_queue: bool) -> Self { let mut claims_per_para = BTreeMap::new(); + let mut claim_queue_state = Vec::with_capacity(group_assignments.len()); + for para_id in group_assignments { *claims_per_para.entry(*para_id).or_default() += 1; + claim_queue_state.push((false, *para_id)); } + // Not optimal but if the claim queue is not available `group_assignments` will have just + // one element. Can be fixed once claim queue api is released everywhere and the fallback + // code is cleaned up. + let claim_queue_state = if has_claim_queue { Some(claim_queue_state) } else { None }; + Self { status: Default::default(), fetching_from: None, @@ -269,7 +279,7 @@ impl Collations { seconded_count: 0, fetched_per_para: Default::default(), claims_per_para, - has_claim_queue, + claim_queue_state, } } @@ -280,7 +290,22 @@ impl Collations { // Note a collation which has been successfully fetched. pub(super) fn note_fetched(&mut self, para_id: ParaId) { - *self.fetched_per_para.entry(para_id).or_default() += 1 + // update the number of fetched collations for the para_id + *self.fetched_per_para.entry(para_id).or_default() += 1; + + // and the claim queue state + if let Some(claim_queue_state) = self.claim_queue_state.as_mut() { + for (satisfied, assignment) in claim_queue_state { + if *satisfied { + continue + } + + if assignment == ¶_id { + *satisfied = true; + break + } + } + } } /// Returns the next collation to fetch from the `waiting_queue`. @@ -356,7 +381,7 @@ impl Collations { self.seconded_count >= 1 }, ProspectiveParachainsMode::Enabled { max_candidate_depth, allowed_ancestry_len: _ } - if !self.has_claim_queue => + if !self.claim_queue_state.is_some() => { gum::trace!( target: LOG_TARGET, @@ -430,50 +455,43 @@ impl Collations { "Pick a collation to fetch." ); - if !self.has_claim_queue { - if let Some(assigned_para_id) = group_assignments.first() { - return self - .waiting_queue - .get_mut(assigned_para_id) - .map(|collations| collations.pop_front()) - .flatten() - } else { - unreachable!("Group assignments should contain at least one element.") - } - } + let claim_queue_state = match self.claim_queue_state.as_mut() { + Some(cqs) => cqs, + // Fallback if claim queue is not avaliable. There is only one assignment in + // `group_assignments` so fetch the first advertisement for it and return. + None => + if let Some(assigned_para_id) = group_assignments.first() { + return self + .waiting_queue + .get_mut(assigned_para_id) + .map(|collations| collations.pop_front()) + .flatten() + } else { + unreachable!("Group assignments should contain at least one element.") + }, + }; - let mut claim_queue_state = Vec::with_capacity(group_assignments.len()); - let mut fetched_per_para = self.fetched_per_para.clone(); let mut pending_fetches = pending_fetches.clone(); - for assignment in group_assignments { - if let Some(fetched) = fetched_per_para.get_mut(assignment) { - if *fetched > 0 { - claim_queue_state.push((true, assignment)); - *fetched -= 1; - continue; - } + for (fulfilled, assignment) in claim_queue_state { + // if this assignment has been already fulfilled - move on + if *fulfilled { + continue } - if let Some(pending_fetches) = pending_fetches.get_mut(assignment) { - if *pending_fetches > 0 { - claim_queue_state.push((true, assignment)); - *pending_fetches -= 1; - continue; + // if there is a pending fetch for this assignment, we should consider it satisfied and + // proceed with the next + if let Some(pending_fetch) = pending_fetches.get_mut(assignment) { + if *pending_fetch > 0 { + *pending_fetch -= 1; + continue } } - claim_queue_state.push((false, assignment)); - } - - for (fulfilled, assignment) in &mut claim_queue_state { - if *fulfilled { - continue - } - + // we have found and unfulfilled assignment - try to fulfill it if let Some(collations) = self.waiting_queue.get_mut(assignment) { if let Some(collation) = collations.pop_front() { - *fulfilled = true; + // we don't mark the entry as fulfilled because it is considered pending return Some(collation) } } From 5f5271219f8501c38a701ccd2cdbca181fda5fcf Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 4 Jul 2024 11:29:55 +0300 Subject: [PATCH 23/55] Remove a redundant runtime version check --- .../src/validator_side/mod.rs | 49 +++++++++++-------- .../src/validator_side/tests/mod.rs | 9 ---- .../tests/prospective_parachains.rs | 17 ------- 3 files changed, 29 insertions(+), 46 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 63062900a0c3..ff6c466b35a3 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -43,13 +43,12 @@ use polkadot_node_subsystem::{ messages::{ CanSecondRequest, CandidateBackingMessage, CollatorProtocolMessage, IfDisconnected, NetworkBridgeEvent, NetworkBridgeTxMessage, ParentHeadData, ProspectiveParachainsMessage, - ProspectiveValidationDataRequest, RuntimeApiRequest, + ProspectiveValidationDataRequest, }, overseer, CollatorProtocolSenderTrait, FromOrchestra, OverseerSignal, PerLeafSpan, }; use polkadot_node_subsystem_util::{ backing_implicit_view::View as ImplicitView, - has_required_runtime, reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL}, runtime::{prospective_parachains_mode, ProspectiveParachainsMode}, vstaging::fetch_claim_queue, @@ -462,13 +461,16 @@ fn is_relay_parent_in_implicit_view( } } +// Returns the group assignments for the validator and bool indicating if they are obtained from the +// claim queue or not. The latter is used to handle the fall back case when the claim queue api is +// not available in the runtime. async fn assign_incoming( sender: &mut Sender, current_assignments: &mut HashMap, keystore: &KeystorePtr, relay_parent: Hash, relay_parent_mode: ProspectiveParachainsMode, -) -> Result +) -> Result<(GroupAssignments, bool)> where Sender: CollatorProtocolSenderTrait, { @@ -495,25 +497,32 @@ where rotation_info.core_for_group(group, cores.len()) } else { gum::trace!(target: LOG_TARGET, ?relay_parent, "Not a validator"); - return Ok(GroupAssignments { current: Vec::new() }) + return Ok((GroupAssignments { current: Vec::new() }, false)) }; - let paras_now = match fetch_claim_queue(sender, relay_parent).await.map_err(Error::Runtime)? { + let (paras_now, has_claim_queue) = match fetch_claim_queue(sender, relay_parent) + .await + .map_err(Error::Runtime)? + { // Runtime supports claim queue - use it // // `relay_parent_mode` is not examined here because if the runtime supports claim queue // then it supports async backing params too (`ASYNC_BACKING_STATE_RUNTIME_REQUIREMENT` // < `CLAIM_QUEUE_RUNTIME_REQUIREMENT`). - Some(mut claim_queue) => claim_queue.0.remove(&core_now), + Some(mut claim_queue) => (claim_queue.0.remove(&core_now), true), // Claim queue is not supported by the runtime - use availability cores instead. - None => cores.get(core_now.0 as usize).and_then(|c| match c { - CoreState::Occupied(core) if relay_parent_mode.is_enabled() => - core.next_up_on_available.as_ref().map(|c| [c.para_id].into_iter().collect()), - CoreState::Scheduled(core) => Some([core.para_id].into_iter().collect()), - CoreState::Occupied(_) | CoreState::Free => None, - }), - } - .unwrap_or_else(|| VecDeque::new()); + None => ( + cores.get(core_now.0 as usize).and_then(|c| match c { + CoreState::Occupied(core) if relay_parent_mode.is_enabled() => + core.next_up_on_available.as_ref().map(|c| [c.para_id].into_iter().collect()), + CoreState::Scheduled(core) => Some([core.para_id].into_iter().collect()), + CoreState::Occupied(_) | CoreState::Free => None, + }), + false, + ), + }; + + let paras_now = paras_now.unwrap_or_else(|| VecDeque::new()); for para_id in paras_now.iter() { let entry = current_assignments.entry(*para_id).or_default(); @@ -528,7 +537,10 @@ where } } - Ok(GroupAssignments { current: paras_now.into_iter().collect::>() }) + Ok(( + GroupAssignments { current: paras_now.into_iter().collect::>() }, + has_claim_queue, + )) } fn remove_outgoing( @@ -1267,16 +1279,13 @@ where for leaf in added { let mode = prospective_parachains_mode(sender, *leaf).await?; - let has_claim_queue_support = - has_required_runtime(sender, *leaf, RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT) - .await; if let Some(span) = view.span_per_head().get(leaf).cloned() { let per_leaf_span = PerLeafSpan::new(span, "validator-side"); state.span_per_relay_parent.insert(*leaf, per_leaf_span); } - let assignments = + let (assignments, has_claim_queue_support) = assign_incoming(sender, &mut state.current_assignments, keystore, *leaf, mode).await?; state.active_leaves.insert(*leaf, mode); @@ -1298,7 +1307,7 @@ where .unwrap_or_default(); for block_hash in allowed_ancestry { if let Entry::Vacant(entry) = state.per_relay_parent.entry(*block_hash) { - let assignments = assign_incoming( + let (assignments, has_claim_queue_support) = assign_incoming( sender, &mut state.current_assignments, keystore, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 5c49de54eb6f..7243af9fe4ca 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -564,15 +564,6 @@ async fn assert_async_backing_params_request(virtual_overseer: &mut VirtualOvers tx.send(Err(ASYNC_BACKING_DISABLED_ERROR)).unwrap(); } ); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::Version(tx), - )) => { - let _ = tx.send(Ok(RuntimeApiRequest::ASYNC_BACKING_STATE_RUNTIME_REQUIREMENT - 1)); - } - ); } // As we receive a relevant advertisement act on it and issue a collation request. diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 91208176747c..64b392dda4f1 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -132,23 +132,6 @@ pub(super) async fn update_view( } ); - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::Version(tx), - )) => { - match test_state.claim_queue { - Some(_) => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); - }, - None => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT - 1)); - } - } - } - ); - assert_assign_incoming( virtual_overseer, test_state, From 6c73e246dbe468b735e745941c7f4d1e98c3352e Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 4 Jul 2024 11:54:57 +0300 Subject: [PATCH 24/55] formatting and comments --- polkadot/node/core/backing/src/lib.rs | 5 +--- .../src/validator_side/collation.rs | 26 ++++++++++++------- .../src/validator_side/mod.rs | 1 - 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 9498e0d657e0..1bda81c5197e 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -281,10 +281,7 @@ impl From<&ActiveLeafState> for ProspectiveParachainsMode { ActiveLeafState::ProspectiveParachainsEnabled { max_candidate_depth, allowed_ancestry_len, - } => ProspectiveParachainsMode::Enabled { - max_candidate_depth, - allowed_ancestry_len, - }, + } => ProspectiveParachainsMode::Enabled { max_candidate_depth, allowed_ancestry_len }, } } } diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 88cfc86be657..3b2efc7fb506 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -247,17 +247,21 @@ pub struct Collations { claims_per_para: BTreeMap, // Represents the claim queue at the relay parent. The `bool` field indicates if a candidate // was fetched for the `ParaId` at the position in question. In other words - if the claim is - // 'satisfied'. If the claim queue is not avaliable `claim_queue_state` will be `None`. + // 'satisfied'. If the claim queue is not available `claim_queue_state` will be `None`. claim_queue_state: Option>, } impl Collations { - /// `Collations` should work with and without claim queue support. To make this happen without - /// creating two parallel implementations instead of working with the claim queue directly it - /// uses `GroupAssignments`. If the runtime supports claim queue `GroupAssignments` contains the - /// claim queue for the core assigned to the (backing group of the) validator. If the runtime - /// doesn't support claim queue `GroupAssignments` contains only one entry - the `ParaId` of the - /// parachain assigned to the core. + /// `Collations` should work with and without claim queue support. If the claim queue runtime + /// api is available `GroupAssignments` the claim queue. If not - group assignments will contain + /// just one item (what's scheduled on the core). + /// + /// Some of the logic in `Collations` relies on the claim queue and if it is not available + /// fallbacks to another logic. For this reason `Collations` needs to know if claim queue is + /// available or not. + /// + /// Once claim queue runtime api is released everywhere this logic won't be needed anymore and + /// can be cleaned up. pub(super) fn new(group_assignments: &Vec, has_claim_queue: bool) -> Self { let mut claims_per_para = BTreeMap::new(); let mut claim_queue_state = Vec::with_capacity(group_assignments.len()); @@ -362,7 +366,7 @@ impl Collations { /// parachains are sharing a core no fairness is guaranteed between them and the faster one can /// starve the slower one by exhausting the limit with its own advertisements. In practice this /// should not happen because core sharing implies core time support which implies the claim - /// queue is available. + /// queue being available. pub(super) fn is_collations_limit_reached( &self, relay_parent_mode: ProspectiveParachainsMode, @@ -441,6 +445,10 @@ impl Collations { /// If claim queue is not supported then `group_assignment` should contain just one element and /// the score won't matter. In this case collations will be fetched in the order they were /// received. + /// + /// Note: `group_assignments` is needed just for the fall back logic. It should be removed once + /// claim queue runtime api is released everywhere since it will be redundant - claim queue will + /// already be available in `self.claim_queue_state`. fn pick_a_collation_to_fetch( &mut self, group_assignments: &Vec, @@ -457,7 +465,7 @@ impl Collations { let claim_queue_state = match self.claim_queue_state.as_mut() { Some(cqs) => cqs, - // Fallback if claim queue is not avaliable. There is only one assignment in + // Fallback if claim queue is not available. There is only one assignment in // `group_assignments` so fetch the first advertisement for it and return. None => if let Some(assigned_para_id) = group_assignments.first() { diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index ff6c466b35a3..71bef5c9d7a9 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -521,7 +521,6 @@ where false, ), }; - let paras_now = paras_now.unwrap_or_else(|| VecDeque::new()); for para_id in paras_now.iter() { From 752f3cc4e7c3e4cb5c5ec10479c39b2b52cc10f8 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 4 Jul 2024 13:47:47 +0300 Subject: [PATCH 25/55] pr doc --- prdoc/pr_4880.prdoc | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 prdoc/pr_4880.prdoc diff --git a/prdoc/pr_4880.prdoc b/prdoc/pr_4880.prdoc new file mode 100644 index 000000000000..863de7570b10 --- /dev/null +++ b/prdoc/pr_4880.prdoc @@ -0,0 +1,25 @@ +title: Collation fetching fairness in collator protocol + +doc: + - audience: "Node Dev" + description: | + Implements collation fetching fairness in the validator side of the collator protocol. With + core time if two (or more) parachains share a single core no fairness is guaranteed between + them in terms of collation fetching. The current implementation was accepting up to + `max_candidate_depth + 1` seconded collations per relay parent and once this limit is reached + no new collations are accepted. A misbehaving collator can abuse this fact and prevent other + collators/parachains to advertise collations by advertising `max_candidate_depth + 1` + collations of its own. + To address this issue two changes are made: + 1. The validator accepts as much advertisements (and collations fetches) as the number of + entries in the claim queue for the parachain in question. + 2. When new collation should be fetched the validator inspects what was fetched so far, what's + in the claim queue and picks the first slot which hasn't got a collation fetched. If there + is a pending advertisement for it it is fetched. Otherwise the next free slot is picked. + These two changes guarantee that: + 1. Validator doesn't accept more collations than it can actually back. + 2. Each parachain has got a fair share of core time based on its allocations. + +crates: + - name: "polkadot-collator-protocol" + bump: "minor" From f0069f106d468291ec5bdb89d6b6c26889fd6781 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 4 Jul 2024 13:56:33 +0300 Subject: [PATCH 26/55] add license --- .../src/validator_side/tests/collation.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 143cdf8fed36..bf6a2e9ca966 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -1,3 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + use std::collections::BTreeMap; use polkadot_node_subsystem_util::runtime::ProspectiveParachainsMode; From 6b9f0b3938870865df02633b7f605135826933dd Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 4 Jul 2024 15:22:49 +0300 Subject: [PATCH 27/55] clippy --- .../src/validator_side/collation.rs | 5 +-- .../src/validator_side/tests/collation.rs | 40 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 3b2efc7fb506..ef83f7622849 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -395,7 +395,7 @@ impl Collations { "is_collations_limit_reached - ProspectiveParachainsMode::Enabled without claim queue support" ); - self.seconded_count >= max_candidate_depth + 1 + self.seconded_count > max_candidate_depth }, ProspectiveParachainsMode::Enabled { max_candidate_depth: _, @@ -472,8 +472,7 @@ impl Collations { return self .waiting_queue .get_mut(assigned_para_id) - .map(|collations| collations.pop_front()) - .flatten() + .and_then(|collations| collations.pop_front()) } else { unreachable!("Group assignments should contain at least one element.") }, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index bf6a2e9ca966..128b9d31038d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -157,7 +157,7 @@ fn collation_fetching_respects_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending ) @@ -169,7 +169,7 @@ fn collation_fetching_respects_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending ) @@ -181,7 +181,7 @@ fn collation_fetching_respects_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending ) @@ -244,7 +244,7 @@ fn collation_fetching_fallback_works() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -256,7 +256,7 @@ fn collation_fetching_fallback_works() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -382,7 +382,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -394,7 +394,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -406,7 +406,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -418,7 +418,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -430,7 +430,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -442,7 +442,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -564,7 +564,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -576,7 +576,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -591,7 +591,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -603,7 +603,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -618,7 +618,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -630,7 +630,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &pending, ) @@ -721,7 +721,7 @@ fn collation_fetching_takes_in_account_pending_items() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &BTreeMap::from([(para_a, 1)]), ) @@ -736,7 +736,7 @@ fn collation_fetching_takes_in_account_pending_items() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &BTreeMap::from([(para_b, 1)]), ) @@ -750,7 +750,7 @@ fn collation_fetching_takes_in_account_pending_items() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode.clone(), + relay_parent_mode, &claim_queue, &BTreeMap::new(), ) From b8c1b8593209b2a6474d5e448b1e6cb127a7acc9 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Fri, 5 Jul 2024 09:53:00 +0300 Subject: [PATCH 28/55] Update prdoc/pr_4880.prdoc Co-authored-by: Maciej --- prdoc/pr_4880.prdoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prdoc/pr_4880.prdoc b/prdoc/pr_4880.prdoc index 863de7570b10..2c77caba846c 100644 --- a/prdoc/pr_4880.prdoc +++ b/prdoc/pr_4880.prdoc @@ -8,7 +8,7 @@ doc: them in terms of collation fetching. The current implementation was accepting up to `max_candidate_depth + 1` seconded collations per relay parent and once this limit is reached no new collations are accepted. A misbehaving collator can abuse this fact and prevent other - collators/parachains to advertise collations by advertising `max_candidate_depth + 1` + collators/parachains from advertising collations by advertising `max_candidate_depth + 1` collations of its own. To address this issue two changes are made: 1. The validator accepts as much advertisements (and collations fetches) as the number of From f26362f64ca0eb3909e0cb1b80b384097ecbae9d Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Sun, 7 Jul 2024 12:56:27 +0300 Subject: [PATCH 29/55] Limit collations based on seconded count instead of number of fetches --- .../src/validator_side/collation.rs | 88 ++++++++++--------- .../src/validator_side/mod.rs | 66 ++++---------- 2 files changed, 60 insertions(+), 94 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index ef83f7622849..8889afac469e 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -192,9 +192,9 @@ pub enum CollationStatus { /// We are waiting for a collation to be advertised to us. Waiting, /// We are currently fetching a collation. - Fetching, + Fetching(ParaId), /// We are waiting that a collation is being validated. - WaitingOnValidation, + WaitingOnValidation(ParaId), /// We have seconded a collation. Seconded, } @@ -237,10 +237,8 @@ pub struct Collations { pub fetching_from: Option<(CollatorId, Option)>, /// Collation that were advertised to us, but we did not yet fetch. Grouped by `ParaId`. waiting_queue: BTreeMap>, - /// How many collations have been seconded. - pub seconded_count: usize, - /// What collations were fetched so far for this relay parent. - fetched_per_para: BTreeMap, + /// How many collations have been seconded per `ParaId`. + seconded_per_para: BTreeMap, // Claims per `ParaId` for the assigned core at the relay parent. This information is obtained // from `GroupAssignments` which contains either the claim queue (if runtime supports it) for // the core or the `ParaId` of the parachain assigned to the core. @@ -280,22 +278,15 @@ impl Collations { status: Default::default(), fetching_from: None, waiting_queue: Default::default(), - seconded_count: 0, - fetched_per_para: Default::default(), + seconded_per_para: Default::default(), claims_per_para, claim_queue_state, } } /// Note a seconded collation for a given para. - pub(super) fn note_seconded(&mut self) { - self.seconded_count += 1 - } - - // Note a collation which has been successfully fetched. - pub(super) fn note_fetched(&mut self, para_id: ParaId) { - // update the number of fetched collations for the para_id - *self.fetched_per_para.entry(para_id).or_default() += 1; + pub(super) fn note_seconded(&mut self, para_id: ParaId) { + *self.seconded_per_para.entry(para_id).or_default() += 1; // and the claim queue state if let Some(claim_queue_state) = self.claim_queue_state.as_mut() { @@ -323,7 +314,6 @@ impl Collations { finished_one: &(CollatorId, Option), relay_parent_mode: ProspectiveParachainsMode, group_assignments: &Vec, - pending_fetches: &BTreeMap, ) -> Option<(PendingCollation, CollatorId)> { // If finished one does not match waiting_collation, then we already dequeued another fetch // to replace it. @@ -348,14 +338,13 @@ impl Collations { // `Waiting` so that we can fetch more collations. If async backing is disabled we can't // fetch more than one collation per relay parent so `None` is returned. CollationStatus::Seconded => None, - CollationStatus::Waiting => - self.pick_a_collation_to_fetch(&group_assignments, pending_fetches), - CollationStatus::WaitingOnValidation | CollationStatus::Fetching => + CollationStatus::Waiting => self.pick_a_collation_to_fetch(&group_assignments), + CollationStatus::WaitingOnValidation(_) | CollationStatus::Fetching(_) => unreachable!("We have reset the status above!"), } } - /// Checks if another collation can be accepted. The number of collations that can be fetched + /// Checks if another collation can be accepted. The number of collations that can be seconded /// per parachain is limited by the entries in claim queue for the `ParaId` in question. /// /// If prospective parachains mode is not enabled then we fall back to synchronous backing. In @@ -371,18 +360,20 @@ impl Collations { &self, relay_parent_mode: ProspectiveParachainsMode, para_id: ParaId, - num_pending_fetches: usize, ) -> bool { + let seconded_for_para = *self.seconded_per_para.get(¶_id).unwrap_or(&0); + let pending_for_para = self.pending_for_para(para_id); + match relay_parent_mode { ProspectiveParachainsMode::Disabled => { gum::trace!( target: LOG_TARGET, ?para_id, - seconded_count=self.seconded_count, + seconded_per_para=?self.seconded_per_para, "is_collations_limit_reached - ProspectiveParachainsMode::Disabled" ); - self.seconded_count >= 1 + seconded_for_para >= 1 }, ProspectiveParachainsMode::Enabled { max_candidate_depth, allowed_ancestry_len: _ } if !self.claim_queue_state.is_some() => @@ -390,12 +381,12 @@ impl Collations { gum::trace!( target: LOG_TARGET, ?para_id, - seconded_count=self.seconded_count, + seconded_per_para=?self.seconded_per_para, max_candidate_depth, "is_collations_limit_reached - ProspectiveParachainsMode::Enabled without claim queue support" ); - self.seconded_count > max_candidate_depth + seconded_for_para > max_candidate_depth }, ProspectiveParachainsMode::Enabled { max_candidate_depth: _, @@ -404,15 +395,14 @@ impl Collations { // Successful fetches + pending fetches < claim queue entries for `para_id` let respected_per_para_limit = self.claims_per_para.get(¶_id).copied().unwrap_or_default() > - self.fetched_per_para.get(¶_id).copied().unwrap_or_default() + - num_pending_fetches; + seconded_for_para + pending_for_para; gum::trace!( target: LOG_TARGET, ?para_id, claims_per_para=?self.claims_per_para, - fetched_per_para=?self.fetched_per_para, - ?num_pending_fetches, + seconded_per_para=?self.seconded_per_para, + ?pending_for_para, ?respected_per_para_limit, "is_collations_limit_reached - ProspectiveParachainsMode::Enabled with claim queue support" ); @@ -432,15 +422,12 @@ impl Collations { /// Picks a collation to fetch from the waiting queue. /// When fetching collations we need to ensure that each parachain has got a fair core time /// share depending on its assignments in the claim queue. This means that the number of - /// collations fetched per parachain should ideally be equal to the number of claims for the + /// collations seconded per parachain should ideally be equal to the number of claims for the /// particular parachain in the claim queue. /// - /// To achieve this each parachain with at an entry in the `waiting_queue` has got a score - /// calculated by dividing the number of fetched collations by the number of entries in the - /// claim queue. Lower score means higher fetching priority. Note that if a parachain hasn't got - /// anything fetched at this relay parent it will have score 0 which means highest priority. If - /// two parachains has got the same score the one which is earlier in the claim queue will be - /// picked. + /// To achieve this each seconded collation is mapped to an entry from the claim queue. The next + /// fetch is the first unsatisfied entry from the claim queue for which there is an + /// advertisement. /// /// If claim queue is not supported then `group_assignment` should contain just one element and /// the score won't matter. In this case collations will be fetched in the order they were @@ -452,12 +439,10 @@ impl Collations { fn pick_a_collation_to_fetch( &mut self, group_assignments: &Vec, - pending_fetches: &BTreeMap, ) -> Option<(PendingCollation, CollatorId)> { gum::trace!( target: LOG_TARGET, waiting_queue=?self.waiting_queue, - fetched_per_para=?self.fetched_per_para, claims_per_para=?self.claims_per_para, ?group_assignments, "Pick a collation to fetch." @@ -478,7 +463,12 @@ impl Collations { }, }; - let mut pending_fetches = pending_fetches.clone(); + let mut pending_for_para = match self.status { + CollationStatus::Waiting => None, + CollationStatus::Fetching(para_id) => Some(para_id), + CollationStatus::WaitingOnValidation(para_id) => Some(para_id), + CollationStatus::Seconded => None, + }; for (fulfilled, assignment) in claim_queue_state { // if this assignment has been already fulfilled - move on @@ -488,9 +478,10 @@ impl Collations { // if there is a pending fetch for this assignment, we should consider it satisfied and // proceed with the next - if let Some(pending_fetch) = pending_fetches.get_mut(assignment) { - if *pending_fetch > 0 { - *pending_fetch -= 1; + if let Some(pending_for) = pending_for_para { + if pending_for == *assignment { + // the pending item should be used only once + pending_for_para = None; continue } } @@ -506,6 +497,17 @@ impl Collations { None } + + // Returns the number of pending collations for the specified `ParaId`. This function should + // return either 0 or 1. + fn pending_for_para(&self, para_id: ParaId) -> usize { + match self.status { + CollationStatus::Fetching(pending_para_id) if pending_para_id == para_id => 1, + CollationStatus::WaitingOnValidation(pending_para_id) if pending_para_id == para_id => + 1, + _ => 0, + } + } } // Any error that can occur when awaiting a collation fetch response. diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 71bef5c9d7a9..ea5b6ecd8e3c 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -19,7 +19,7 @@ use futures::{ }; use futures_timer::Delay; use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap, HashSet, VecDeque}, + collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, future::Future, time::{Duration, Instant}, }; @@ -751,7 +751,7 @@ async fn request_collation( let maybe_candidate_hash = prospective_candidate.as_ref().map(ProspectiveCandidate::candidate_hash); - per_relay_parent.collations.status = CollationStatus::Fetching; + per_relay_parent.collations.status = CollationStatus::Fetching(para_id); per_relay_parent .collations .fetching_from @@ -1112,11 +1112,10 @@ where ) .map_err(AdvertisementError::Invalid)?; - if per_relay_parent.collations.is_collations_limit_reached( - relay_parent_mode, - para_id, - num_pending_collations_for_para_at_relay_parent(&state, para_id, relay_parent), - ) { + if per_relay_parent + .collations + .is_collations_limit_reached(relay_parent_mode, para_id) + { return Err(AdvertisementError::SecondedLimitReached) } @@ -1186,8 +1185,6 @@ where "Received advertise collation", ); - let num_pending_fetches = - num_pending_collations_for_para_at_relay_parent(&state, para_id, relay_parent); let per_relay_parent = match state.per_relay_parent.get_mut(&relay_parent) { Some(rp_state) => rp_state, None => { @@ -1211,7 +1208,7 @@ where }); let collations = &mut per_relay_parent.collations; - if collations.is_collations_limit_reached(relay_parent_mode, para_id, num_pending_fetches) { + if collations.is_collations_limit_reached(relay_parent_mode, para_id) { gum::trace!( target: LOG_TARGET, peer_id = ?peer_id, @@ -1226,7 +1223,7 @@ where PendingCollation::new(relay_parent, para_id, &peer_id, prospective_candidate); match collations.status { - CollationStatus::Fetching | CollationStatus::WaitingOnValidation => { + CollationStatus::Fetching(_) | CollationStatus::WaitingOnValidation(_) => { gum::trace!( target: LOG_TARGET, peer_id = ?peer_id, @@ -1501,8 +1498,9 @@ async fn process_msg( if let Some(CollationEvent { collator_id, pending_collation, .. }) = state.fetched_candidates.remove(&fetched_collation) { - let PendingCollation { relay_parent, peer_id, prospective_candidate, .. } = - pending_collation; + let PendingCollation { + relay_parent, peer_id, prospective_candidate, para_id, .. + } = pending_collation; note_good_collation( &mut state.reputation, ctx.sender(), @@ -1523,7 +1521,7 @@ async fn process_msg( if let Some(rp_state) = state.per_relay_parent.get_mut(&parent) { rp_state.collations.status = CollationStatus::Seconded; - rp_state.collations.note_seconded(); + rp_state.collations.note_seconded(para_id); } // See if we've unblocked other collations for seconding. @@ -1671,10 +1669,6 @@ async fn run_inner( let CollationEvent {collator_id, pending_collation, .. } = res.collation_event.clone(); - state.per_relay_parent.get_mut(&pending_collation.relay_parent).map(|rp_state| { - rp_state.collations.note_fetched(pending_collation.para_id); - }); - match kick_off_seconding(&mut ctx, &mut state, res).await { Err(err) => { gum::warn!( @@ -1746,14 +1740,12 @@ async fn dequeue_next_collation_and_fetch( // The collator we tried to fetch from last, optionally which candidate. previous_fetch: (CollatorId, Option), ) { - let pending_collations = pending_collations_per_para_at_relay_parent(state, relay_parent); while let Some((next, id)) = state.per_relay_parent.get_mut(&relay_parent).and_then(|rp_state| { rp_state.collations.get_next_collation_to_fetch( &previous_fetch, rp_state.prospective_parachains_mode, &rp_state.assignment.current, - &pending_collations, ) }) { gum::debug!( @@ -1938,6 +1930,8 @@ async fn kick_off_seconding( maybe_parent_head.and_then(|head| maybe_parent_head_hash.map(|hash| (head, hash))), )?; + let para_id = candidate_receipt.descriptor().para_id; + ctx.send_message(CandidateBackingMessage::Second( relay_parent, candidate_receipt, @@ -1947,7 +1941,7 @@ async fn kick_off_seconding( .await; // There's always a single collation being fetched at any moment of time. // In case of a failure, we reset the status back to waiting. - collations.status = CollationStatus::WaitingOnValidation; + collations.status = CollationStatus::WaitingOnValidation(para_id); entry.insert(collation_event); Ok(true) @@ -2127,33 +2121,3 @@ async fn handle_collation_fetch_response( state.metrics.on_request(metrics_result); result } - -// Returns the number of pending fetches for `ParaId` at the specified relay parent. -fn num_pending_collations_for_para_at_relay_parent( - state: &State, - para_id: ParaId, - relay_parent: Hash, -) -> usize { - state - .collation_requests_cancel_handles - .iter() - .filter(|(pending_collation, _)| { - pending_collation.para_id == para_id && pending_collation.relay_parent == relay_parent - }) - .count() -} - -// Returns the number of pending fetches for each `ParaId` at the specified relay parent. -fn pending_collations_per_para_at_relay_parent( - state: &State, - relay_parent: Hash, -) -> BTreeMap { - state - .collation_requests_cancel_handles - .iter() - .filter(|(col, _)| col.relay_parent == relay_parent) - .fold(BTreeMap::new(), |mut res, (col, _)| { - *res.entry(col.para_id).or_default() += 1; - res - }) -} From d6857fc564c8e3e9c5e79bebbb0766d5b9e414e4 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Sun, 7 Jul 2024 13:00:37 +0300 Subject: [PATCH 30/55] Undo rename: is_seconded_limit_reached --- .../src/validator_side/collation.rs | 8 ++++---- .../src/validator_side/mod.rs | 6 +++--- .../src/validator_side/tests/collation.rs | 20 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 8889afac469e..2b30b72e714e 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -356,7 +356,7 @@ impl Collations { /// starve the slower one by exhausting the limit with its own advertisements. In practice this /// should not happen because core sharing implies core time support which implies the claim /// queue being available. - pub(super) fn is_collations_limit_reached( + pub(super) fn is_seconded_limit_reached( &self, relay_parent_mode: ProspectiveParachainsMode, para_id: ParaId, @@ -370,7 +370,7 @@ impl Collations { target: LOG_TARGET, ?para_id, seconded_per_para=?self.seconded_per_para, - "is_collations_limit_reached - ProspectiveParachainsMode::Disabled" + "is_seconded_limit_reached - ProspectiveParachainsMode::Disabled" ); seconded_for_para >= 1 @@ -383,7 +383,7 @@ impl Collations { ?para_id, seconded_per_para=?self.seconded_per_para, max_candidate_depth, - "is_collations_limit_reached - ProspectiveParachainsMode::Enabled without claim queue support" + "is_seconded_limit_reached - ProspectiveParachainsMode::Enabled without claim queue support" ); seconded_for_para > max_candidate_depth @@ -404,7 +404,7 @@ impl Collations { seconded_per_para=?self.seconded_per_para, ?pending_for_para, ?respected_per_para_limit, - "is_collations_limit_reached - ProspectiveParachainsMode::Enabled with claim queue support" + "is_seconded_limit_reached - ProspectiveParachainsMode::Enabled with claim queue support" ); !respected_per_para_limit diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index ea5b6ecd8e3c..a842f54821ce 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -1114,7 +1114,7 @@ where if per_relay_parent .collations - .is_collations_limit_reached(relay_parent_mode, para_id) + .is_seconded_limit_reached(relay_parent_mode, para_id) { return Err(AdvertisementError::SecondedLimitReached) } @@ -1208,7 +1208,7 @@ where }); let collations = &mut per_relay_parent.collations; - if collations.is_collations_limit_reached(relay_parent_mode, para_id) { + if collations.is_seconded_limit_reached(relay_parent_mode, para_id) { gum::trace!( target: LOG_TARGET, peer_id = ?peer_id, @@ -1239,7 +1239,7 @@ where fetch_collation(sender, state, pending_collation, collator_id).await?; }, CollationStatus::Seconded if relay_parent_mode.is_enabled() => { - // Limit is not reached (checked with `is_collations_limit_reached` before the match + // Limit is not reached (checked with `is_seconded_limit_reached` before the match // expression), it's allowed to second another collation. fetch_collation(sender, state, pending_collation, collator_id).await?; }, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 128b9d31038d..9de08f856a18 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -38,28 +38,28 @@ fn cant_add_more_than_claim_queue() { let mut collations = Collations::new(&assignments, claim_queue_support); // first collation for `para_a` is in the limit - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 0,)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0,)); collations.note_fetched(para_a); // and `para_b` is not affected - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); // second collation for `para_a` is also in the limit - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0)); collations.note_fetched(para_a); // `para_b`` is still not affected - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); // third collation for `para_a`` will be above the limit - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0)); // one fetch for b - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); collations.note_fetched(para_b); // and now both paras are over limit - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a, 0)); - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_b, 0)); + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0)); + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); } #[test] @@ -78,11 +78,11 @@ fn pending_fetches_are_counted() { collations.fetching_from = Some((collator_id_a, None)); // first collation for `para_a` is in the limit - assert!(!collations.is_collations_limit_reached(relay_parent_mode, para_a, 1)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a, 1)); collations.note_fetched(para_a); // second collation for `para_a`` is not in the limit due to the pending fetch - assert!(collations.is_collations_limit_reached(relay_parent_mode, para_a, 1)); + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a, 1)); } #[test] From cde28cd9cb425d5111bc4b1da54360e3719acc70 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 8 Jul 2024 10:00:42 +0300 Subject: [PATCH 31/55] fix collation tests --- .../src/validator_side/tests/collation.rs | 219 ++++-------------- 1 file changed, 42 insertions(+), 177 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 9de08f856a18..cdfdcafcd8b1 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -14,14 +14,14 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use std::collections::BTreeMap; - use polkadot_node_subsystem_util::runtime::ProspectiveParachainsMode; use polkadot_primitives::{CandidateHash, CollatorId, Hash, Id as ParaId}; use sc_network::PeerId; use sp_core::sr25519; +use crate::validator_side::tests::CollationStatus; + use super::{Collations, PendingCollation, ProspectiveCandidate}; #[test] @@ -38,28 +38,28 @@ fn cant_add_more_than_claim_queue() { let mut collations = Collations::new(&assignments, claim_queue_support); // first collation for `para_a` is in the limit - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0,)); - collations.note_fetched(para_a); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + collations.note_seconded(para_a); // and `para_b` is not affected - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); // second collation for `para_a` is also in the limit - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0)); - collations.note_fetched(para_a); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + collations.note_seconded(para_a); // `para_b`` is still not affected - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); // third collation for `para_a`` will be above the limit - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0)); + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a)); // one fetch for b - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); - collations.note_fetched(para_b); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); + collations.note_seconded(para_b); // and now both paras are over limit - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a, 0)); - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_b, 0)); + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_b)); } #[test] @@ -67,7 +67,6 @@ fn pending_fetches_are_counted() { sp_tracing::init_for_tests(); let para_a = ParaId::from(1); - let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; let relay_parent_mode = @@ -75,14 +74,17 @@ fn pending_fetches_are_counted() { let claim_queue_support = true; let mut collations = Collations::new(&assignments, claim_queue_support); - collations.fetching_from = Some((collator_id_a, None)); + collations.status = CollationStatus::Fetching(para_a); //para_a is pending // first collation for `para_a` is in the limit - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a, 1)); - collations.note_fetched(para_a); + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + collations.note_seconded(para_a); + + // second collation for `para_a` is not in the limit due to the pending fetch + assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a)); - // second collation for `para_a`` is not in the limit due to the pending fetch - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a, 1)); + // a collation for `para_b` is accepted since the pending fetch is for `para_a` + assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); } #[test] @@ -101,11 +103,11 @@ fn collation_fetching_respects_claim_queue() { let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; let claim_queue_support = true; - let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; + collations.status = CollationStatus::Waiting; //nothing pending let relay_parent = Hash::repeat_byte(0x01); @@ -159,10 +161,9 @@ fn collation_fetching_respects_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending ) ); - collations.note_fetched(collation_a1.0.para_id); + collations.note_seconded(collation_a1.0.para_id); assert_eq!( Some(collation_b1.clone()), @@ -171,10 +172,9 @@ fn collation_fetching_respects_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending ) ); - collations.note_fetched(collation_b1.0.para_id); + collations.note_seconded(collation_b1.0.para_id); assert_eq!( Some(collation_a2.clone()), @@ -183,10 +183,9 @@ fn collation_fetching_respects_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending ) ); - collations.note_fetched(collation_a2.0.para_id); + collations.note_seconded(collation_a2.0.para_id); } #[test] @@ -201,11 +200,11 @@ fn collation_fetching_fallback_works() { let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; let claim_queue_support = false; - let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; + collations.status = CollationStatus::Waiting; //nothing pending let relay_parent = Hash::repeat_byte(0x01); @@ -246,10 +245,9 @@ fn collation_fetching_fallback_works() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_a1.0.para_id); + collations.note_seconded(collation_a1.0.para_id); assert_eq!( Some(collation_a2.clone()), @@ -258,10 +256,9 @@ fn collation_fetching_fallback_works() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_a2.0.para_id); + collations.note_seconded(collation_a2.0.para_id); } #[test] @@ -284,10 +281,10 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; - let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; + collations.status = CollationStatus::Waiting; //nothing pending let relay_parent = Hash::repeat_byte(0x01); @@ -384,10 +381,9 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_a1.0.para_id); + collations.note_seconded(collation_a1.0.para_id); assert_eq!( Some(collation_b1.clone()), @@ -396,10 +392,9 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_b1.0.para_id); + collations.note_seconded(collation_b1.0.para_id); assert_eq!( Some(collation_a2.clone()), @@ -408,10 +403,9 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_a2.0.para_id); + collations.note_seconded(collation_a2.0.para_id); assert_eq!( Some(collation_b2.clone()), @@ -420,10 +414,9 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_b2.0.para_id); + collations.note_seconded(collation_b2.0.para_id); assert_eq!( Some(collation_c1.clone()), @@ -432,10 +425,9 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_c1.0.para_id); + collations.note_seconded(collation_c1.0.para_id); assert_eq!( Some(collation_c2.clone()), @@ -444,10 +436,9 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_c2.0.para_id); + collations.note_seconded(collation_c2.0.para_id); } #[test] @@ -470,10 +461,10 @@ fn collation_fetching_fills_holes_in_claim_queue() { let relay_parent_mode = ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; - let pending = BTreeMap::new(); let mut collations = Collations::new(&claim_queue, claim_queue_support); collations.fetching_from = None; + collations.status = CollationStatus::Waiting; //nothing pending let relay_parent = Hash::repeat_byte(0x01); @@ -566,10 +557,9 @@ fn collation_fetching_fills_holes_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_a1.0.para_id); + collations.note_seconded(collation_a1.0.para_id); assert_eq!( Some(collation_c1.clone()), @@ -578,10 +568,9 @@ fn collation_fetching_fills_holes_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_c1.0.para_id); + collations.note_seconded(collation_c1.0.para_id); collations.add_to_waiting_queue(collation_c2.clone()); collations.add_to_waiting_queue(collation_b1.clone()); @@ -593,10 +582,9 @@ fn collation_fetching_fills_holes_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_b1.0.para_id); + collations.note_seconded(collation_b1.0.para_id); assert_eq!( Some(collation_c2.clone()), @@ -605,10 +593,9 @@ fn collation_fetching_fills_holes_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, ) ); - collations.note_fetched(collation_c2.0.para_id); + collations.note_seconded(collation_c2.0.para_id); collations.add_to_waiting_queue(collation_b2.clone()); collations.add_to_waiting_queue(collation_a2.clone()); @@ -620,131 +607,10 @@ fn collation_fetching_fills_holes_in_claim_queue() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &pending, - ) - ); - collations.note_fetched(collation_a2.0.para_id); - - assert_eq!( - Some(collation_b2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, - &claim_queue, - &pending, - ) - ); - collations.note_fetched(collation_b2.0.para_id); -} - -#[test] -fn collation_fetching_takes_in_account_pending_items() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); - let peer_a = PeerId::random(); - - let para_b = ParaId::from(2); - let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); - let peer_b = PeerId::random(); - - let claim_queue = vec![para_a, para_b, para_a, para_b]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 5, allowed_ancestry_len: 4 }; - let claim_queue_support = true; - - let mut collations = Collations::new(&claim_queue, claim_queue_support); - collations.fetching_from = None; - - let relay_parent = Hash::repeat_byte(0x01); - - let collation_a1 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(1)), - parent_head_data_hash: Hash::repeat_byte(1), - }), - ), - collator_id_a.clone(), - ); - - let collation_a2 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(2)), - parent_head_data_hash: Hash::repeat_byte(2), - }), - ), - collator_id_a.clone(), - ); - - let collation_b1 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(3)), - parent_head_data_hash: Hash::repeat_byte(3), - }), - ), - collator_id_b.clone(), - ); - - let collation_b2 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(4)), - parent_head_data_hash: Hash::repeat_byte(4), - }), - ), - collator_id_b.clone(), - ); - - // a1 will be pending, a2 and b1 will be in the queue; b1 should be fetched first - collations.add_to_waiting_queue(collation_a2.clone()); - collations.add_to_waiting_queue(collation_b1.clone()); - - assert_eq!( - Some(collation_b1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, - &claim_queue, - &BTreeMap::from([(para_a, 1)]), - ) - ); - collations.note_fetched(collation_a1.0.para_id); // a1 is no longer pending - - // a1 is fetched, b1 is pending, a2 and b2 are in the queue, a2 should be fetched next - collations.add_to_waiting_queue(collation_b2.clone()); - - assert_eq!( - Some(collation_a2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, - &claim_queue, - &BTreeMap::from([(para_b, 1)]), ) ); - collations.note_fetched(collation_b1.0.para_id); - collations.note_fetched(collation_a2.0.para_id); + collations.note_seconded(collation_a2.0.para_id); - // and finally b2 should be fetched assert_eq!( Some(collation_b2.clone()), collations.get_next_collation_to_fetch( @@ -752,8 +618,7 @@ fn collation_fetching_takes_in_account_pending_items() { &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), relay_parent_mode, &claim_queue, - &BTreeMap::new(), ) ); - collations.note_fetched(collation_b2.0.para_id); + collations.note_seconded(collation_b2.0.para_id); } From 4c3db2ad607371bb375ef54dc21620fc4d6c130e Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 8 Jul 2024 10:19:40 +0300 Subject: [PATCH 32/55] `collations_fetching_respects_seconded_limit` test --- .../src/validator_side/tests/collation.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index cdfdcafcd8b1..56f950a85ca3 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -622,3 +622,35 @@ fn collation_fetching_fills_holes_in_claim_queue() { ); collations.note_seconded(collation_b2.0.para_id); } + +#[test] +fn collations_fetching_respects_seconded_limit() { + sp_tracing::init_for_tests(); + + let para_a = ParaId::from(1); + let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); + + let para_b = ParaId::from(2); + + let claim_queue = vec![para_a, para_b, para_a]; + let relay_parent_mode = + ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; + let claim_queue_support = true; + + let mut collations = Collations::new(&claim_queue, claim_queue_support); + collations.fetching_from = None; + collations.status = CollationStatus::Fetching(para_a); //para_a is pending + + collations.note_seconded(para_a); + collations.note_seconded(para_a); + + assert_eq!( + None, + collations.get_next_collation_to_fetch( + // doesn't matter since `fetching_from` is `None` + &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), + relay_parent_mode, + &claim_queue, + ) + ); +} From b2bbdfe5352538fbf11377d818efbbe45c86ee63 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 8 Jul 2024 10:44:11 +0300 Subject: [PATCH 33/55] nits --- .../src/validator_side/collation.rs | 6 +++--- .../src/validator_side/tests/collation.rs | 8 +++++--- prdoc/pr_4880.prdoc | 14 ++++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 2b30b72e714e..fadba3c03557 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -244,7 +244,7 @@ pub struct Collations { // the core or the `ParaId` of the parachain assigned to the core. claims_per_para: BTreeMap, // Represents the claim queue at the relay parent. The `bool` field indicates if a candidate - // was fetched for the `ParaId` at the position in question. In other words - if the claim is + // was seconded for the `ParaId` at the position in question. In other words - if the claim is // 'satisfied'. If the claim queue is not available `claim_queue_state` will be `None`. claim_queue_state: Option>, } @@ -362,7 +362,6 @@ impl Collations { para_id: ParaId, ) -> bool { let seconded_for_para = *self.seconded_per_para.get(¶_id).unwrap_or(&0); - let pending_for_para = self.pending_for_para(para_id); match relay_parent_mode { ProspectiveParachainsMode::Disabled => { @@ -392,7 +391,8 @@ impl Collations { max_candidate_depth: _, allowed_ancestry_len: _, } => { - // Successful fetches + pending fetches < claim queue entries for `para_id` + let pending_for_para = self.pending_for_para(para_id); + let respected_per_para_limit = self.claims_per_para.get(¶_id).copied().unwrap_or_default() > seconded_for_para + pending_for_para; diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 56f950a85ca3..6d641ba7bf50 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -150,9 +150,9 @@ fn collation_fetching_respects_claim_queue() { collator_id_b.clone(), ); + collations.add_to_waiting_queue(collation_b1.clone()); collations.add_to_waiting_queue(collation_a1.clone()); collations.add_to_waiting_queue(collation_a2.clone()); - collations.add_to_waiting_queue(collation_b1.clone()); assert_eq!( Some(collation_a1.clone()), @@ -366,7 +366,7 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collator_id_c.clone(), ); - // Despite the order here the fetches should be a1, b1, c1, a2, b2, c2 + // Despite the order here the fetches should follow the claim queue collations.add_to_waiting_queue(collation_c1.clone()); collations.add_to_waiting_queue(collation_c2.clone()); collations.add_to_waiting_queue(collation_b1.clone()); @@ -546,7 +546,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { collator_id_c.clone(), ); - // Despite the order here the fetches should be a1, b1, c1, a2, b2, c2 collations.add_to_waiting_queue(collation_c1.clone()); collations.add_to_waiting_queue(collation_a1.clone()); @@ -561,6 +560,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { ); collations.note_seconded(collation_a1.0.para_id); + // fetch c1 since there is nothing better to fetch assert_eq!( Some(collation_c1.clone()), collations.get_next_collation_to_fetch( @@ -572,6 +572,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { ); collations.note_seconded(collation_c1.0.para_id); + // b1 should be prioritized since there is a hole in the claim queue collations.add_to_waiting_queue(collation_c2.clone()); collations.add_to_waiting_queue(collation_b1.clone()); @@ -597,6 +598,7 @@ fn collation_fetching_fills_holes_in_claim_queue() { ); collations.note_seconded(collation_c2.0.para_id); + // same with a2 collations.add_to_waiting_queue(collation_b2.clone()); collations.add_to_waiting_queue(collation_a2.clone()); diff --git a/prdoc/pr_4880.prdoc b/prdoc/pr_4880.prdoc index 2c77caba846c..3b16081f5dd1 100644 --- a/prdoc/pr_4880.prdoc +++ b/prdoc/pr_4880.prdoc @@ -11,14 +11,16 @@ doc: collators/parachains from advertising collations by advertising `max_candidate_depth + 1` collations of its own. To address this issue two changes are made: - 1. The validator accepts as much advertisements (and collations fetches) as the number of - entries in the claim queue for the parachain in question. - 2. When new collation should be fetched the validator inspects what was fetched so far, what's - in the claim queue and picks the first slot which hasn't got a collation fetched. If there - is a pending advertisement for it it is fetched. Otherwise the next free slot is picked. + 1. For each parachain id the validator accepts advertisements until the number of entries in + the claim queue equals the number of seconded candidates. + 2. When new collation should be fetched the validator inspects what was seconded so far, + what's in the claim queue and picks the first slot which hasn't got a collation seconded + and there is no candidate pending seconding for it. If there is an advertisement in the + waiting queue for it it is fetched. Otherwise the next free slot is picked. These two changes guarantee that: 1. Validator doesn't accept more collations than it can actually back. - 2. Each parachain has got a fair share of core time based on its allocations. + 2. Each parachain has got a fair share of core time based on its allocations in the claim + queue. crates: - name: "polkadot-collator-protocol" From 01d121e997c4e3bbb22c76e20a8b2f7268b8aa5f Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 26 Aug 2024 12:51:03 +0300 Subject: [PATCH 34/55] Remove duplicated dependency after merge --- polkadot/node/network/collator-protocol/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/node/network/collator-protocol/Cargo.toml b/polkadot/node/network/collator-protocol/Cargo.toml index 9c3e08b76148..8a7c384dcbe8 100644 --- a/polkadot/node/network/collator-protocol/Cargo.toml +++ b/polkadot/node/network/collator-protocol/Cargo.toml @@ -36,7 +36,6 @@ rstest = { workspace = true } sp-core = { features = ["std"], workspace = true, default-features = true } sp-keyring = { workspace = true, default-features = true } sc-keystore = { workspace = true, default-features = true } -sp-tracing = { workspace = true } sc-network = { workspace = true, default-features = true } codec = { features = ["std"], workspace = true, default-features = true } From 7b3c002e05da3ad112d6562452cdcd52f366a1db Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 10 Jul 2024 11:07:26 +0300 Subject: [PATCH 35/55] Remove `ProspectiveParachainsMode` from collator-protocol, validator-side --- .../src/validator_side/collation.rs | 88 ++---- .../src/validator_side/mod.rs | 296 +++++++----------- 2 files changed, 146 insertions(+), 238 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index fadba3c03557..27bc1b2e966b 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -42,9 +42,7 @@ use polkadot_node_network_protocol::{ }; use polkadot_node_primitives::PoV; use polkadot_node_subsystem::jaeger; -use polkadot_node_subsystem_util::{ - metrics::prometheus::prometheus::HistogramTimer, runtime::ProspectiveParachainsMode, -}; +use polkadot_node_subsystem_util::metrics::prometheus::prometheus::HistogramTimer; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorId, Hash, HeadData, Id as ParaId, PersistedValidationData, @@ -207,16 +205,9 @@ impl Default for CollationStatus { impl CollationStatus { /// Downgrades to `Waiting`, but only if `self != Seconded`. - fn back_to_waiting(&mut self, relay_parent_mode: ProspectiveParachainsMode) { - match self { - Self::Seconded => - if relay_parent_mode.is_enabled() { - // With async backing enabled it's allowed to - // second more candidates. - *self = Self::Waiting - }, - _ => *self = Self::Waiting, - } + fn back_to_waiting(&mut self) { + // With async backing it's allowed to second more candidates. + *self = Self::Waiting } } @@ -312,7 +303,6 @@ impl Collations { pub(super) fn get_next_collation_to_fetch( &mut self, finished_one: &(CollatorId, Option), - relay_parent_mode: ProspectiveParachainsMode, group_assignments: &Vec, ) -> Option<(PendingCollation, CollatorId)> { // If finished one does not match waiting_collation, then we already dequeued another fetch @@ -331,7 +321,7 @@ impl Collations { return None } } - self.status.back_to_waiting(relay_parent_mode); + self.status.back_to_waiting(); match self.status { // If async backing is enabled `back_to_waiting` will change `Seconded` state to @@ -358,58 +348,40 @@ impl Collations { /// queue being available. pub(super) fn is_seconded_limit_reached( &self, - relay_parent_mode: ProspectiveParachainsMode, + max_candidate_depth: u32, para_id: ParaId, ) -> bool { let seconded_for_para = *self.seconded_per_para.get(¶_id).unwrap_or(&0); - match relay_parent_mode { - ProspectiveParachainsMode::Disabled => { - gum::trace!( - target: LOG_TARGET, - ?para_id, - seconded_per_para=?self.seconded_per_para, - "is_seconded_limit_reached - ProspectiveParachainsMode::Disabled" - ); + if !self.claim_queue_state.is_some() { + gum::trace!( + target: LOG_TARGET, + ?para_id, + seconded_per_para=?self.seconded_per_para, + max_candidate_depth, + "is_seconded_limit_reached - no claim queue support" + ); - seconded_for_para >= 1 - }, - ProspectiveParachainsMode::Enabled { max_candidate_depth, allowed_ancestry_len: _ } - if !self.claim_queue_state.is_some() => - { - gum::trace!( - target: LOG_TARGET, - ?para_id, - seconded_per_para=?self.seconded_per_para, - max_candidate_depth, - "is_seconded_limit_reached - ProspectiveParachainsMode::Enabled without claim queue support" - ); + return seconded_for_para > max_candidate_depth as usize; + } - seconded_for_para > max_candidate_depth - }, - ProspectiveParachainsMode::Enabled { - max_candidate_depth: _, - allowed_ancestry_len: _, - } => { - let pending_for_para = self.pending_for_para(para_id); + let pending_for_para = self.pending_for_para(para_id); - let respected_per_para_limit = - self.claims_per_para.get(¶_id).copied().unwrap_or_default() > - seconded_for_para + pending_for_para; + let respected_per_para_limit = + self.claims_per_para.get(¶_id).copied().unwrap_or_default() > + seconded_for_para + pending_for_para; - gum::trace!( - target: LOG_TARGET, - ?para_id, - claims_per_para=?self.claims_per_para, - seconded_per_para=?self.seconded_per_para, - ?pending_for_para, - ?respected_per_para_limit, - "is_seconded_limit_reached - ProspectiveParachainsMode::Enabled with claim queue support" - ); + gum::trace!( + target: LOG_TARGET, + ?para_id, + claims_per_para=?self.claims_per_para, + seconded_per_para=?self.seconded_per_para, + ?pending_for_para, + ?respected_per_para_limit, + "is_seconded_limit_reached - with claim queue support" + ); - !respected_per_para_limit - }, - } + !respected_per_para_limit } /// Adds a new collation to the waiting queue for the relay parent. This function doesn't diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index a842f54821ce..5be6004b666d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -50,12 +50,13 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_util::{ backing_implicit_view::View as ImplicitView, reputation::{ReputationAggregator, REPUTATION_CHANGE_INTERVAL}, - runtime::{prospective_parachains_mode, ProspectiveParachainsMode}, + request_async_backing_params, + runtime::recv_runtime, vstaging::fetch_claim_queue, }; use polkadot_primitives::{ - CandidateHash, CollatorId, CoreState, Hash, HeadData, Id as ParaId, OccupiedCoreAssumption, - PersistedValidationData, + AsyncBackingParams, CandidateHash, CollatorId, CoreState, Hash, HeadData, Id as ParaId, + OccupiedCoreAssumption, PersistedValidationData, }; use crate::error::{Error, FetchError, Result, SecondingError}; @@ -163,8 +164,7 @@ impl PeerData { fn update_view( &mut self, implicit_view: &ImplicitView, - active_leaves: &HashMap, - per_relay_parent: &HashMap, + active_leaves: &HashMap, new_view: View, ) { let old_view = std::mem::replace(&mut self.view, new_view); @@ -172,18 +172,12 @@ impl PeerData { for removed in old_view.difference(&self.view) { // Remove relay parent advertisements if it went out // of our (implicit) view. - let keep = per_relay_parent - .get(removed) - .map(|s| { - is_relay_parent_in_implicit_view( - removed, - s.prospective_parachains_mode, - implicit_view, - active_leaves, - peer_state.para_id, - ) - }) - .unwrap_or(false); + let keep = is_relay_parent_in_implicit_view( + removed, + implicit_view, + active_leaves, + peer_state.para_id, + ); if !keep { peer_state.advertisements.remove(&removed); @@ -196,8 +190,7 @@ impl PeerData { fn prune_old_advertisements( &mut self, implicit_view: &ImplicitView, - active_leaves: &HashMap, - per_relay_parent: &HashMap, + active_leaves: &HashMap, ) { if let PeerState::Collating(ref mut peer_state) = self.state { peer_state.advertisements.retain(|hash, _| { @@ -205,15 +198,12 @@ impl PeerData { // - Relay parent is an active leaf // - It belongs to allowed ancestry under some leaf // Discard otherwise. - per_relay_parent.get(hash).map_or(false, |s| { - is_relay_parent_in_implicit_view( - hash, - s.prospective_parachains_mode, - implicit_view, - active_leaves, - peer_state.para_id, - ) - }) + is_relay_parent_in_implicit_view( + hash, + implicit_view, + active_leaves, + peer_state.para_id, + ) }); } } @@ -224,17 +214,15 @@ impl PeerData { fn insert_advertisement( &mut self, on_relay_parent: Hash, - relay_parent_mode: ProspectiveParachainsMode, candidate_hash: Option, implicit_view: &ImplicitView, - active_leaves: &HashMap, + active_leaves: &HashMap, ) -> std::result::Result<(CollatorId, ParaId), InsertAdvertisementError> { match self.state { PeerState::Connected(_) => Err(InsertAdvertisementError::UndeclaredCollator), PeerState::Collating(ref mut state) => { if !is_relay_parent_in_implicit_view( &on_relay_parent, - relay_parent_mode, implicit_view, active_leaves, state.para_id, @@ -242,47 +230,34 @@ impl PeerData { return Err(InsertAdvertisementError::OutOfOurView) } - match (relay_parent_mode, candidate_hash) { - (ProspectiveParachainsMode::Disabled, candidate_hash) => { - if state.advertisements.contains_key(&on_relay_parent) { - return Err(InsertAdvertisementError::Duplicate) - } - state - .advertisements - .insert(on_relay_parent, HashSet::from_iter(candidate_hash)); - }, - (ProspectiveParachainsMode::Enabled { .. }, candidate_hash) => { - if let Some(candidate_hash) = candidate_hash { - if state - .advertisements - .get(&on_relay_parent) - .map_or(false, |candidates| candidates.contains(&candidate_hash)) - { - return Err(InsertAdvertisementError::Duplicate) - } - - let candidates = - state.advertisements.entry(on_relay_parent).or_default(); - - candidates.insert(candidate_hash); - } else { - if self.version != CollationVersion::V1 { - gum::error!( - target: LOG_TARGET, - "Programming error, `candidate_hash` can not be `None` \ - for non `V1` networking.", - ); - } - - if state.advertisements.contains_key(&on_relay_parent) { - return Err(InsertAdvertisementError::Duplicate) - } - state - .advertisements - .insert(on_relay_parent, HashSet::from_iter(candidate_hash)); - }; - }, - } + if let Some(candidate_hash) = candidate_hash { + if state + .advertisements + .get(&on_relay_parent) + .map_or(false, |candidates| candidates.contains(&candidate_hash)) + { + return Err(InsertAdvertisementError::Duplicate) + } + + let candidates = state.advertisements.entry(on_relay_parent).or_default(); + + candidates.insert(candidate_hash); + } else { + if self.version != CollationVersion::V1 { + gum::error!( + target: LOG_TARGET, + "Programming error, `candidate_hash` can not be `None` \ + for non `V1` networking.", + ); + } + + if state.advertisements.contains_key(&on_relay_parent) { + return Err(InsertAdvertisementError::Duplicate) + } + state + .advertisements + .insert(on_relay_parent, HashSet::from_iter(candidate_hash)); + }; state.last_active = Instant::now(); Ok((state.collator_id.clone(), state.para_id)) @@ -363,19 +338,19 @@ struct GroupAssignments { } struct PerRelayParent { - prospective_parachains_mode: ProspectiveParachainsMode, + async_backing_params: AsyncBackingParams, assignment: GroupAssignments, collations: Collations, } impl PerRelayParent { fn new( - mode: ProspectiveParachainsMode, + async_backing_params: AsyncBackingParams, assignments: GroupAssignments, has_claim_queue: bool, ) -> Self { let collations = Collations::new(&assignments.current, has_claim_queue); - Self { prospective_parachains_mode: mode, assignment: assignments, collations } + Self { async_backing_params, assignment: assignments, collations } } } @@ -396,7 +371,7 @@ struct State { /// support prospective parachains. This mapping works as a replacement for /// [`polkadot_node_network_protocol::View`] and can be dropped once the transition /// to asynchronous backing is done. - active_leaves: HashMap, + active_leaves: HashMap, /// State tracked per relay parent. per_relay_parent: HashMap, @@ -444,21 +419,16 @@ struct State { fn is_relay_parent_in_implicit_view( relay_parent: &Hash, - relay_parent_mode: ProspectiveParachainsMode, implicit_view: &ImplicitView, - active_leaves: &HashMap, + active_leaves: &HashMap, para_id: ParaId, ) -> bool { - match relay_parent_mode { - ProspectiveParachainsMode::Disabled => active_leaves.contains_key(relay_parent), - ProspectiveParachainsMode::Enabled { .. } => active_leaves.iter().any(|(hash, mode)| { - mode.is_enabled() && - implicit_view - .known_allowed_relay_parents_under(hash, Some(para_id)) - .unwrap_or_default() - .contains(relay_parent) - }), - } + active_leaves.iter().any(|(hash, _)| { + implicit_view + .known_allowed_relay_parents_under(hash, Some(para_id)) + .unwrap_or_default() + .contains(relay_parent) + }) } // Returns the group assignments for the validator and bool indicating if they are obtained from the @@ -469,7 +439,6 @@ async fn assign_incoming( current_assignments: &mut HashMap, keystore: &KeystorePtr, relay_parent: Hash, - relay_parent_mode: ProspectiveParachainsMode, ) -> Result<(GroupAssignments, bool)> where Sender: CollatorProtocolSenderTrait, @@ -505,18 +474,14 @@ where .map_err(Error::Runtime)? { // Runtime supports claim queue - use it - // - // `relay_parent_mode` is not examined here because if the runtime supports claim queue - // then it supports async backing params too (`ASYNC_BACKING_STATE_RUNTIME_REQUIREMENT` - // < `CLAIM_QUEUE_RUNTIME_REQUIREMENT`). Some(mut claim_queue) => (claim_queue.0.remove(&core_now), true), // Claim queue is not supported by the runtime - use availability cores instead. None => ( cores.get(core_now.0 as usize).and_then(|c| match c { - CoreState::Occupied(core) if relay_parent_mode.is_enabled() => + CoreState::Occupied(core) => core.next_up_on_available.as_ref().map(|c| [c.para_id].into_iter().collect()), CoreState::Scheduled(core) => Some([core.para_id].into_iter().collect()), - CoreState::Occupied(_) | CoreState::Free => None, + CoreState::Free => None, }), false, ), @@ -663,12 +628,7 @@ fn handle_peer_view_change(state: &mut State, peer_id: PeerId, view: View) { None => return, }; - peer_data.update_view( - &state.implicit_view, - &state.active_leaves, - &state.per_relay_parent, - view, - ); + peer_data.update_view(&state.implicit_view, &state.active_leaves, view); state.collation_requests_cancel_handles.retain(|pc, handle| { let keep = pc.peer_id != peer_id || peer_data.has_advertised(&pc.relay_parent, None); if !keep { @@ -1089,7 +1049,6 @@ where .get(&relay_parent) .ok_or(AdvertisementError::RelayParentUnknown)?; - let relay_parent_mode = per_relay_parent.prospective_parachains_mode; let assignment = &per_relay_parent.assignment; let collator_para_id = @@ -1105,17 +1064,16 @@ where let (collator_id, para_id) = peer_data .insert_advertisement( relay_parent, - relay_parent_mode, candidate_hash, &state.implicit_view, &state.active_leaves, ) .map_err(AdvertisementError::Invalid)?; - if per_relay_parent - .collations - .is_seconded_limit_reached(relay_parent_mode, para_id) - { + if per_relay_parent.collations.is_seconded_limit_reached( + per_relay_parent.async_backing_params.max_candidate_depth, + para_id, + ) { return Err(AdvertisementError::SecondedLimitReached) } @@ -1123,15 +1081,14 @@ where // Check if backing subsystem allows to second this candidate. // // This is also only important when async backing or elastic scaling is enabled. - let seconding_not_allowed = relay_parent_mode.is_enabled() && - !can_second( - sender, - collator_para_id, - relay_parent, - candidate_hash, - parent_head_data_hash, - ) - .await; + let seconding_not_allowed = !can_second( + sender, + collator_para_id, + relay_parent, + candidate_hash, + parent_head_data_hash, + ) + .await; if seconding_not_allowed { return Err(AdvertisementError::BlockedByBacking) @@ -1200,7 +1157,6 @@ where return Ok(()) }, }; - let relay_parent_mode = per_relay_parent.prospective_parachains_mode; let prospective_candidate = prospective_candidate.map(|(candidate_hash, parent_head_data_hash)| ProspectiveCandidate { candidate_hash, @@ -1208,7 +1164,10 @@ where }); let collations = &mut per_relay_parent.collations; - if collations.is_seconded_limit_reached(relay_parent_mode, para_id) { + if collations.is_seconded_limit_reached( + per_relay_parent.async_backing_params.max_candidate_depth, + para_id, + ) { gum::trace!( target: LOG_TARGET, peer_id = ?peer_id, @@ -1238,21 +1197,11 @@ where // the new collation immediately fetch_collation(sender, state, pending_collation, collator_id).await?; }, - CollationStatus::Seconded if relay_parent_mode.is_enabled() => { + CollationStatus::Seconded => { // Limit is not reached (checked with `is_seconded_limit_reached` before the match // expression), it's allowed to second another collation. fetch_collation(sender, state, pending_collation, collator_id).await?; }, - CollationStatus::Seconded => { - gum::trace!( - target: LOG_TARGET, - peer_id = ?peer_id, - %para_id, - ?relay_parent, - ?relay_parent_mode, - "A collation has already been seconded", - ); - }, } Ok(()) @@ -1274,7 +1223,8 @@ where let added = view.iter().filter(|h| !current_leaves.contains_key(h)); for leaf in added { - let mode = prospective_parachains_mode(sender, *leaf).await?; + let async_backing_params = + recv_runtime(request_async_backing_params(*leaf, sender).await).await?; if let Some(span) = view.span_per_head().get(leaf).cloned() { let per_leaf_span = PerLeafSpan::new(span, "validator-side"); @@ -1282,52 +1232,46 @@ where } let (assignments, has_claim_queue_support) = - assign_incoming(sender, &mut state.current_assignments, keystore, *leaf, mode).await?; + assign_incoming(sender, &mut state.current_assignments, keystore, *leaf).await?; - state.active_leaves.insert(*leaf, mode); - state - .per_relay_parent - .insert(*leaf, PerRelayParent::new(mode, assignments, has_claim_queue_support)); - - if mode.is_enabled() { - state - .implicit_view - .activate_leaf(sender, *leaf) - .await - .map_err(Error::ImplicitViewFetchError)?; - - // Order is always descending. - let allowed_ancestry = state - .implicit_view - .known_allowed_relay_parents_under(leaf, None) - .unwrap_or_default(); - for block_hash in allowed_ancestry { - if let Entry::Vacant(entry) = state.per_relay_parent.entry(*block_hash) { - let (assignments, has_claim_queue_support) = assign_incoming( - sender, - &mut state.current_assignments, - keystore, - *block_hash, - mode, - ) - .await?; + state.active_leaves.insert(*leaf, async_backing_params); + state.per_relay_parent.insert( + *leaf, + PerRelayParent::new(async_backing_params, assignments, has_claim_queue_support), + ); - entry.insert(PerRelayParent::new(mode, assignments, has_claim_queue_support)); - } + state + .implicit_view + .activate_leaf(sender, *leaf) + .await + .map_err(Error::ImplicitViewFetchError)?; + + // Order is always descending. + let allowed_ancestry = state + .implicit_view + .known_allowed_relay_parents_under(leaf, None) + .unwrap_or_default(); + for block_hash in allowed_ancestry { + if let Entry::Vacant(entry) = state.per_relay_parent.entry(*block_hash) { + let (assignments, has_claim_queue_support) = + assign_incoming(sender, &mut state.current_assignments, keystore, *block_hash) + .await?; + + entry.insert(PerRelayParent::new( + async_backing_params, + assignments, + has_claim_queue_support, + )); } } } - for (removed, mode) in removed { + for (removed, _) in removed { state.active_leaves.remove(removed); // If the leaf is deactivated it still may stay in the view as a part // of implicit ancestry. Only update the state after the hash is actually // pruned from the block info storage. - let pruned = if mode.is_enabled() { - state.implicit_view.deactivate_leaf(*removed) - } else { - vec![*removed] - }; + let pruned = state.implicit_view.deactivate_leaf(*removed); for removed in pruned { if let Some(per_relay_parent) = state.per_relay_parent.remove(&removed) { @@ -1358,11 +1302,7 @@ where }); for (peer_id, peer_data) in state.peer_data.iter_mut() { - peer_data.prune_old_advertisements( - &state.implicit_view, - &state.active_leaves, - &state.per_relay_parent, - ); + peer_data.prune_old_advertisements(&state.implicit_view, &state.active_leaves); // Disconnect peers who are not relevant to our current or next para. // @@ -1742,11 +1682,9 @@ async fn dequeue_next_collation_and_fetch( ) { while let Some((next, id)) = state.per_relay_parent.get_mut(&relay_parent).and_then(|rp_state| { - rp_state.collations.get_next_collation_to_fetch( - &previous_fetch, - rp_state.prospective_parachains_mode, - &rp_state.assignment.current, - ) + rp_state + .collations + .get_next_collation_to_fetch(&previous_fetch, &rp_state.assignment.current) }) { gum::debug!( target: LOG_TARGET, @@ -1852,9 +1790,7 @@ async fn kick_off_seconding( collation_event.collator_protocol_version, collation_event.pending_collation.prospective_candidate, ) { - (CollationVersion::V2, Some(ProspectiveCandidate { parent_head_data_hash, .. })) - if per_relay_parent.prospective_parachains_mode.is_enabled() => - { + (CollationVersion::V2, Some(ProspectiveCandidate { parent_head_data_hash, .. })) => { let pvd = request_prospective_validation_data( ctx.sender(), relay_parent, @@ -1867,7 +1803,7 @@ async fn kick_off_seconding( (pvd, maybe_parent_head_data, Some(parent_head_data_hash)) }, // Support V2 collators without async backing enabled. - (CollationVersion::V2, Some(_)) | (CollationVersion::V1, _) => { + (CollationVersion::V1, _) => { let pvd = request_persisted_validation_data( ctx.sender(), candidate_receipt.descriptor().relay_parent, From 5dffddefcacc6573f2e18262d806c8a31741184a Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 10 Jul 2024 10:36:41 +0300 Subject: [PATCH 36/55] Fix compilation errors in collation tests --- .../src/validator_side/tests/collation.rs | 57 +++++-------------- 1 file changed, 13 insertions(+), 44 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs index 6d641ba7bf50..24972e66926d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs @@ -14,7 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_node_subsystem_util::runtime::ProspectiveParachainsMode; use polkadot_primitives::{CandidateHash, CollatorId, Hash, Id as ParaId}; use sc_network::PeerId; @@ -31,35 +30,34 @@ fn cant_add_more_than_claim_queue() { let para_a = ParaId::from(1); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let max_candidate_depth = 4; let claim_queue_support = true; let mut collations = Collations::new(&assignments, claim_queue_support); // first collation for `para_a` is in the limit - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_a)); collations.note_seconded(para_a); // and `para_b` is not affected - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); + assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); // second collation for `para_a` is also in the limit - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_a)); collations.note_seconded(para_a); // `para_b`` is still not affected - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); + assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); // third collation for `para_a`` will be above the limit - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_a)); // one fetch for b - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); + assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); collations.note_seconded(para_b); // and now both paras are over limit - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a)); - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_b)); + assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_a)); + assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_b)); } #[test] @@ -69,22 +67,21 @@ fn pending_fetches_are_counted() { let para_a = ParaId::from(1); let para_b = ParaId::from(2); let assignments = vec![para_a, para_b, para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; + let max_candidate_depth = 4; let claim_queue_support = true; let mut collations = Collations::new(&assignments, claim_queue_support); collations.status = CollationStatus::Fetching(para_a); //para_a is pending // first collation for `para_a` is in the limit - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_a)); collations.note_seconded(para_a); // second collation for `para_a` is not in the limit due to the pending fetch - assert!(collations.is_seconded_limit_reached(relay_parent_mode, para_a)); + assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_a)); // a collation for `para_b` is accepted since the pending fetch is for `para_a` - assert!(!collations.is_seconded_limit_reached(relay_parent_mode, para_b)); + assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); } #[test] @@ -100,8 +97,6 @@ fn collation_fetching_respects_claim_queue() { let peer_b = PeerId::random(); let claim_queue = vec![para_a, para_b, para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; let claim_queue_support = true; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -159,7 +154,6 @@ fn collation_fetching_respects_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -170,7 +164,6 @@ fn collation_fetching_respects_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -181,7 +174,6 @@ fn collation_fetching_respects_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -197,8 +189,6 @@ fn collation_fetching_fallback_works() { let peer_a = PeerId::random(); let claim_queue = vec![para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 4, allowed_ancestry_len: 3 }; let claim_queue_support = false; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -243,7 +233,6 @@ fn collation_fetching_fallback_works() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -254,7 +243,6 @@ fn collation_fetching_fallback_works() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -278,8 +266,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { let peer_c = PeerId::random(); let claim_queue = vec![para_a, para_b, para_a, para_b, para_c, para_c]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -379,7 +365,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -390,7 +375,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -401,7 +385,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -412,7 +395,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -423,7 +405,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -434,7 +415,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -458,8 +438,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { let peer_c = PeerId::random(); let claim_queue = vec![para_a, para_b, para_a, para_b, para_c, para_c]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -554,7 +532,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -566,7 +543,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -581,7 +557,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -592,7 +567,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -607,7 +581,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -618,7 +591,6 @@ fn collation_fetching_fills_holes_in_claim_queue() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); @@ -635,8 +607,6 @@ fn collations_fetching_respects_seconded_limit() { let para_b = ParaId::from(2); let claim_queue = vec![para_a, para_b, para_a]; - let relay_parent_mode = - ProspectiveParachainsMode::Enabled { max_candidate_depth: 7, allowed_ancestry_len: 6 }; let claim_queue_support = true; let mut collations = Collations::new(&claim_queue, claim_queue_support); @@ -651,7 +621,6 @@ fn collations_fetching_respects_seconded_limit() { collations.get_next_collation_to_fetch( // doesn't matter since `fetching_from` is `None` &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - relay_parent_mode, &claim_queue, ) ); From 1c1744b23207cdcd08b3dfc2b3bb65bc9fb0884a Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 11 Jul 2024 13:41:56 +0300 Subject: [PATCH 37/55] `is_seconded_limit_reached` uses the whole view --- .../src/validator_side/collation.rs | 158 ++------------- .../src/validator_side/mod.rs | 185 +++++++++++++----- 2 files changed, 156 insertions(+), 187 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 27bc1b2e966b..ae063b2b6c72 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -193,8 +193,6 @@ pub enum CollationStatus { Fetching(ParaId), /// We are waiting that a collation is being validated. WaitingOnValidation(ParaId), - /// We have seconded a collation. - Seconded, } impl Default for CollationStatus { @@ -203,14 +201,6 @@ impl Default for CollationStatus { } } -impl CollationStatus { - /// Downgrades to `Waiting`, but only if `self != Seconded`. - fn back_to_waiting(&mut self) { - // With async backing it's allowed to second more candidates. - *self = Self::Waiting - } -} - /// Information about collations per relay parent. pub struct Collations { /// What is the current status in regards to a collation for this relay parent? @@ -294,96 +284,6 @@ impl Collations { } } - /// Returns the next collation to fetch from the `waiting_queue`. - /// - /// This will reset the status back to `Waiting` using [`CollationStatus::back_to_waiting`]. - /// - /// Returns `Some(_)` if there is any collation to fetch, the `status` is not `Seconded` and - /// the passed in `finished_one` is the currently `waiting_collation`. - pub(super) fn get_next_collation_to_fetch( - &mut self, - finished_one: &(CollatorId, Option), - group_assignments: &Vec, - ) -> Option<(PendingCollation, CollatorId)> { - // If finished one does not match waiting_collation, then we already dequeued another fetch - // to replace it. - if let Some((collator_id, maybe_candidate_hash)) = self.fetching_from.as_ref() { - // If a candidate hash was saved previously, `finished_one` must include this too. - if collator_id != &finished_one.0 && - maybe_candidate_hash.map_or(true, |hash| Some(&hash) != finished_one.1.as_ref()) - { - gum::trace!( - target: LOG_TARGET, - waiting_collation = ?self.fetching_from, - ?finished_one, - "Not proceeding to the next collation - has already been done." - ); - return None - } - } - self.status.back_to_waiting(); - - match self.status { - // If async backing is enabled `back_to_waiting` will change `Seconded` state to - // `Waiting` so that we can fetch more collations. If async backing is disabled we can't - // fetch more than one collation per relay parent so `None` is returned. - CollationStatus::Seconded => None, - CollationStatus::Waiting => self.pick_a_collation_to_fetch(&group_assignments), - CollationStatus::WaitingOnValidation(_) | CollationStatus::Fetching(_) => - unreachable!("We have reset the status above!"), - } - } - - /// Checks if another collation can be accepted. The number of collations that can be seconded - /// per parachain is limited by the entries in claim queue for the `ParaId` in question. - /// - /// If prospective parachains mode is not enabled then we fall back to synchronous backing. In - /// this case there is a limit of 1 collation per relay parent. - /// - /// If prospective parachains mode is enabled but claim queue is not supported then up to - /// `max_candidate_depth + 1` seconded collations are accepted. In theory in this case if two - /// parachains are sharing a core no fairness is guaranteed between them and the faster one can - /// starve the slower one by exhausting the limit with its own advertisements. In practice this - /// should not happen because core sharing implies core time support which implies the claim - /// queue being available. - pub(super) fn is_seconded_limit_reached( - &self, - max_candidate_depth: u32, - para_id: ParaId, - ) -> bool { - let seconded_for_para = *self.seconded_per_para.get(¶_id).unwrap_or(&0); - - if !self.claim_queue_state.is_some() { - gum::trace!( - target: LOG_TARGET, - ?para_id, - seconded_per_para=?self.seconded_per_para, - max_candidate_depth, - "is_seconded_limit_reached - no claim queue support" - ); - - return seconded_for_para > max_candidate_depth as usize; - } - - let pending_for_para = self.pending_for_para(para_id); - - let respected_per_para_limit = - self.claims_per_para.get(¶_id).copied().unwrap_or_default() > - seconded_for_para + pending_for_para; - - gum::trace!( - target: LOG_TARGET, - ?para_id, - claims_per_para=?self.claims_per_para, - seconded_per_para=?self.seconded_per_para, - ?pending_for_para, - ?respected_per_para_limit, - "is_seconded_limit_reached - with claim queue support" - ); - - !respected_per_para_limit - } - /// Adds a new collation to the waiting queue for the relay parent. This function doesn't /// perform any limits check. The caller (`enqueue_collation`) should assure that the collation /// limit is respected. @@ -408,58 +308,25 @@ impl Collations { /// Note: `group_assignments` is needed just for the fall back logic. It should be removed once /// claim queue runtime api is released everywhere since it will be redundant - claim queue will /// already be available in `self.claim_queue_state`. - fn pick_a_collation_to_fetch( + pub(super) fn pick_a_collation_to_fetch( &mut self, - group_assignments: &Vec, + claim_queue_state: Vec<(bool, ParaId)>, ) -> Option<(PendingCollation, CollatorId)> { gum::trace!( target: LOG_TARGET, waiting_queue=?self.waiting_queue, claims_per_para=?self.claims_per_para, - ?group_assignments, "Pick a collation to fetch." ); - let claim_queue_state = match self.claim_queue_state.as_mut() { - Some(cqs) => cqs, - // Fallback if claim queue is not available. There is only one assignment in - // `group_assignments` so fetch the first advertisement for it and return. - None => - if let Some(assigned_para_id) = group_assignments.first() { - return self - .waiting_queue - .get_mut(assigned_para_id) - .and_then(|collations| collations.pop_front()) - } else { - unreachable!("Group assignments should contain at least one element.") - }, - }; - - let mut pending_for_para = match self.status { - CollationStatus::Waiting => None, - CollationStatus::Fetching(para_id) => Some(para_id), - CollationStatus::WaitingOnValidation(para_id) => Some(para_id), - CollationStatus::Seconded => None, - }; - for (fulfilled, assignment) in claim_queue_state { // if this assignment has been already fulfilled - move on - if *fulfilled { + if fulfilled { continue } - // if there is a pending fetch for this assignment, we should consider it satisfied and - // proceed with the next - if let Some(pending_for) = pending_for_para { - if pending_for == *assignment { - // the pending item should be used only once - pending_for_para = None; - continue - } - } - // we have found and unfulfilled assignment - try to fulfill it - if let Some(collations) = self.waiting_queue.get_mut(assignment) { + if let Some(collations) = self.waiting_queue.get_mut(&assignment) { if let Some(collation) = collations.pop_front() { // we don't mark the entry as fulfilled because it is considered pending return Some(collation) @@ -472,14 +339,25 @@ impl Collations { // Returns the number of pending collations for the specified `ParaId`. This function should // return either 0 or 1. - fn pending_for_para(&self, para_id: ParaId) -> usize { + fn pending_for_para(&self, para_id: &ParaId) -> usize { match self.status { - CollationStatus::Fetching(pending_para_id) if pending_para_id == para_id => 1, - CollationStatus::WaitingOnValidation(pending_para_id) if pending_para_id == para_id => + CollationStatus::Fetching(pending_para_id) if pending_para_id == *para_id => 1, + CollationStatus::WaitingOnValidation(pending_para_id) + if pending_para_id == *para_id => 1, _ => 0, } } + + // Returns the number of seconded collations for the specified `ParaId`. + pub(super) fn seconded_and_pending_for_para(&self, para_id: &ParaId) -> usize { + *self.seconded_per_para.get(¶_id).unwrap_or(&0) + self.pending_for_para(para_id) + } + + // Returns the number of claims in the claim queue for the specified `ParaId`. + pub(super) fn claims_for_para(&self, para_id: &ParaId) -> usize { + self.claims_per_para.get(para_id).copied().unwrap_or_default() + } } // Any error that can occur when awaiting a collation fetch response. diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 5be6004b666d..6438410f149f 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -338,19 +338,14 @@ struct GroupAssignments { } struct PerRelayParent { - async_backing_params: AsyncBackingParams, assignment: GroupAssignments, collations: Collations, } impl PerRelayParent { - fn new( - async_backing_params: AsyncBackingParams, - assignments: GroupAssignments, - has_claim_queue: bool, - ) -> Self { + fn new(assignments: GroupAssignments, has_claim_queue: bool) -> Self { let collations = Collations::new(&assignments.current, has_claim_queue); - Self { async_backing_params, assignment: assignments, collations } + Self { assignment: assignments, collations } } } @@ -415,6 +410,9 @@ struct State { /// Aggregated reputation change reputation: ReputationAggregator, + + /// Last known finalized block number + last_finalized_block_num: u32, } fn is_relay_parent_in_implicit_view( @@ -1070,9 +1068,12 @@ where ) .map_err(AdvertisementError::Invalid)?; - if per_relay_parent.collations.is_seconded_limit_reached( - per_relay_parent.async_backing_params.max_candidate_depth, - para_id, + if is_seconded_limit_reached( + &state.implicit_view, + &state.per_relay_parent, + relay_parent, + per_relay_parent, + ¶_id, ) { return Err(AdvertisementError::SecondedLimitReached) } @@ -1121,7 +1122,7 @@ where } /// Enqueue collation for fetching. The advertisement is expected to be -/// validated. +/// validated and the seconding limit checked. async fn enqueue_collation( sender: &mut Sender, state: &mut State, @@ -1141,7 +1142,6 @@ where ?relay_parent, "Received advertise collation", ); - let per_relay_parent = match state.per_relay_parent.get_mut(&relay_parent) { Some(rp_state) => rp_state, None => { @@ -1164,20 +1164,6 @@ where }); let collations = &mut per_relay_parent.collations; - if collations.is_seconded_limit_reached( - per_relay_parent.async_backing_params.max_candidate_depth, - para_id, - ) { - gum::trace!( - target: LOG_TARGET, - peer_id = ?peer_id, - %para_id, - ?relay_parent, - "Limit of seconded collations reached for valid advertisement", - ); - return Ok(()) - } - let pending_collation = PendingCollation::new(relay_parent, para_id, &peer_id, prospective_candidate); @@ -1197,11 +1183,6 @@ where // the new collation immediately fetch_collation(sender, state, pending_collation, collator_id).await?; }, - CollationStatus::Seconded => { - // Limit is not reached (checked with `is_seconded_limit_reached` before the match - // expression), it's allowed to second another collation. - fetch_collation(sender, state, pending_collation, collator_id).await?; - }, } Ok(()) @@ -1235,10 +1216,9 @@ where assign_incoming(sender, &mut state.current_assignments, keystore, *leaf).await?; state.active_leaves.insert(*leaf, async_backing_params); - state.per_relay_parent.insert( - *leaf, - PerRelayParent::new(async_backing_params, assignments, has_claim_queue_support), - ); + state + .per_relay_parent + .insert(*leaf, PerRelayParent::new(assignments, has_claim_queue_support)); state .implicit_view @@ -1257,11 +1237,7 @@ where assign_incoming(sender, &mut state.current_assignments, keystore, *block_hash) .await?; - entry.insert(PerRelayParent::new( - async_backing_params, - assignments, - has_claim_queue_support, - )); + entry.insert(PerRelayParent::new(assignments, has_claim_queue_support)); } } } @@ -1288,6 +1264,8 @@ where state.fetched_candidates.retain(|k, _| k.relay_parent != removed); state.span_per_relay_parent.remove(&removed); } + + state.last_finalized_block_num = view.finalized_number; } // Remove blocked seconding requests that left the view. @@ -1460,7 +1438,7 @@ async fn process_msg( } if let Some(rp_state) = state.per_relay_parent.get_mut(&parent) { - rp_state.collations.status = CollationStatus::Seconded; + rp_state.collations.status = CollationStatus::Waiting; rp_state.collations.note_seconded(para_id); } @@ -1680,12 +1658,7 @@ async fn dequeue_next_collation_and_fetch( // The collator we tried to fetch from last, optionally which candidate. previous_fetch: (CollatorId, Option), ) { - while let Some((next, id)) = - state.per_relay_parent.get_mut(&relay_parent).and_then(|rp_state| { - rp_state - .collations - .get_next_collation_to_fetch(&previous_fetch, &rp_state.assignment.current) - }) { + while let Some((next, id)) = get_next_collation_to_fetch(&previous_fetch, relay_parent, state) { gum::debug!( target: LOG_TARGET, ?relay_parent, @@ -2057,3 +2030,121 @@ async fn handle_collation_fetch_response( state.metrics.on_request(metrics_result); result } + +/// Checks if another collation can be accepted. The number of collations that can be seconded +/// per parachain is limited by the entries in claim queue for the `ParaId` in question. Besides the +/// seconded collations at the relay parent of the advertisement any pending or seconded collations +/// at previous relay parents (up to `allowed_ancestry_len` blocks back or last finalized block) are +/// also counted towards the limit. +fn is_seconded_limit_reached( + implicit_view: &ImplicitView, + per_relay_parent: &HashMap, + relay_parent: Hash, + relay_parent_state: &PerRelayParent, + para_id: &ParaId, +) -> bool { + // Get the number of claims for `para_id` at `relay_parent` + // TODO: should be compared against the number of items in the claim queue!? + let claims_for_para = relay_parent_state.collations.claims_for_para(para_id); + + let seconded_and_pending_at_ancestors = seconded_and_pending_for_para_in_view( + implicit_view, + per_relay_parent, + &relay_parent, + para_id, + ); + + claims_for_para >= seconded_and_pending_at_ancestors +} + +fn seconded_and_pending_for_para_in_view( + implicit_view: &ImplicitView, + per_relay_parent: &HashMap, + relay_parent: &Hash, + para_id: &ParaId, +) -> usize { + // `known_allowed_relay_parents_under` returns all leaves within the view for the specified + // block hash including the block hash itself + implicit_view + .known_allowed_relay_parents_under(relay_parent, Some(*para_id)) + .map(|ancestors| { + ancestors.iter().fold(0, |res, anc| { + res + per_relay_parent + .get(&anc) + .map(|rp| rp.collations.seconded_and_pending_for_para(para_id)) + .unwrap_or(0) + }) + }) + .unwrap_or(0) +} + +fn claim_queue_state( + relay_parent: &Hash, + per_relay_parent: &HashMap, + implicit_view: &ImplicitView, +) -> Option> { + let relay_parent_state = per_relay_parent.get(relay_parent)?; + let scheduled_paras = relay_parent_state.assignment.current.iter().collect::>(); + let mut claims_per_para = scheduled_paras + .into_iter() + .map(|para_id| { + ( + *para_id, + seconded_and_pending_for_para_in_view( + implicit_view, + per_relay_parent, + relay_parent, + para_id, + ), + ) + }) + .collect::>(); + let claim_queue_state = relay_parent_state + .assignment + .current + .iter() + .map(|para_id| match claims_per_para.entry(*para_id) { + Entry::Occupied(mut entry) if *entry.get() > 0 => { + *entry.get_mut() -= 1; + (true, *para_id) + }, + _ => (false, *para_id), + }) + .collect::>(); + Some(claim_queue_state) +} + +/// Returns the next collation to fetch from the `waiting_queue`. +/// +/// This will reset the status back to `Waiting` using [`CollationStatus::back_to_waiting`]. +/// +/// Returns `Some(_)` if there is any collation to fetch, the `status` is not `Seconded` and +/// the passed in `finished_one` is the currently `waiting_collation`. +fn get_next_collation_to_fetch( + finished_one: &(CollatorId, Option), + relay_parent: Hash, + state: &mut State, +) -> Option<(PendingCollation, CollatorId)> { + let claim_queue_state = + claim_queue_state(&relay_parent, &state.per_relay_parent, &state.implicit_view)?; + let rp_state = state.per_relay_parent.get_mut(&relay_parent)?; // TODO: this is looked up twice + + // If finished one does not match waiting_collation, then we already dequeued another fetch + // to replace it. + if let Some((collator_id, maybe_candidate_hash)) = rp_state.collations.fetching_from.as_ref() { + // If a candidate hash was saved previously, `finished_one` must include this too. + if collator_id != &finished_one.0 && + maybe_candidate_hash.map_or(true, |hash| Some(&hash) != finished_one.1.as_ref()) + { + gum::trace!( + target: LOG_TARGET, + waiting_collation = ?rp_state.collations.fetching_from, + ?finished_one, + "Not proceeding to the next collation - has already been done." + ); + return None + } + } + rp_state.collations.status = CollationStatus::Waiting; + rp_state.collations.pick_a_collation_to_fetch(claim_queue_state) +} From aaccab1de9905611f5cd902ec00bee5820a66ae7 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Fri, 30 Aug 2024 11:54:08 +0300 Subject: [PATCH 38/55] Fix `is_seconded_limit_reached` check --- .../collator-protocol/src/validator_side/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 6438410f149f..1870a85b5fb2 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -2043,10 +2043,7 @@ fn is_seconded_limit_reached( relay_parent_state: &PerRelayParent, para_id: &ParaId, ) -> bool { - // Get the number of claims for `para_id` at `relay_parent` - // TODO: should be compared against the number of items in the claim queue!? let claims_for_para = relay_parent_state.collations.claims_for_para(para_id); - let seconded_and_pending_at_ancestors = seconded_and_pending_for_para_in_view( implicit_view, per_relay_parent, @@ -2054,7 +2051,16 @@ fn is_seconded_limit_reached( para_id, ); - claims_for_para >= seconded_and_pending_at_ancestors + gum::trace!( + target: LOG_TARGET, + ?relay_parent, + ?para_id, + claims_for_para, + seconded_and_pending_at_ancestors, + "Checking if seconded limit is reached" + ); + + seconded_and_pending_at_ancestors >= claims_for_para } fn seconded_and_pending_for_para_in_view( From b1df2e32eb3541b43158aa96b1c8c55a28232e9f Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 11 Sep 2024 10:37:19 +0300 Subject: [PATCH 39/55] Trace logs useful for debugging tests --- .../src/validator_side/collation.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index ae063b2b6c72..a31b640668e5 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -269,6 +269,8 @@ impl Collations { pub(super) fn note_seconded(&mut self, para_id: ParaId) { *self.seconded_per_para.entry(para_id).or_default() += 1; + gum::trace!(target: LOG_TARGET, ?para_id, new_count=*self.seconded_per_para.entry(para_id).or_default(), "Note seconded."); + // and the claim queue state if let Some(claim_queue_state) = self.claim_queue_state.as_mut() { for (satisfied, assignment) in claim_queue_state { @@ -351,7 +353,18 @@ impl Collations { // Returns the number of seconded collations for the specified `ParaId`. pub(super) fn seconded_and_pending_for_para(&self, para_id: &ParaId) -> usize { - *self.seconded_per_para.get(¶_id).unwrap_or(&0) + self.pending_for_para(para_id) + let seconded_for_para = *self.seconded_per_para.get(¶_id).unwrap_or(&0); + let pending_for_para = self.pending_for_para(para_id); + + gum::trace!( + target: LOG_TARGET, + ?para_id, + seconded_for_para, + pending_for_para, + "Seconded and pending for para." + ); + + seconded_for_para + pending_for_para } // Returns the number of claims in the claim queue for the specified `ParaId`. From ce3a95e470db9a30ddbea4e61b8bdc087d9c8d02 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 11 Sep 2024 10:50:17 +0300 Subject: [PATCH 40/55] Handle unconnected candidates --- .../network/collator-protocol/src/validator_side/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 1870a85b5fb2..a1c403541210 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -2070,9 +2070,15 @@ fn seconded_and_pending_for_para_in_view( para_id: &ParaId, ) -> usize { // `known_allowed_relay_parents_under` returns all leaves within the view for the specified - // block hash including the block hash itself + // block hash including the block hash itself. + // If the relay parent is not in the view (unconnected candidate) + // `known_allowed_relay_parents_under` will return `None`. In this case we still we just count + // the candidate at the specified relay parent. + // TODO: what to do when an unconnected candidate becomes connected and potentially we have + // accepted more candidates than the claim queue allows? implicit_view .known_allowed_relay_parents_under(relay_parent, Some(*para_id)) + .or(Some(&[*relay_parent])) // if the relay parent is not in view we still want to count it .map(|ancestors| { ancestors.iter().fold(0, |res, anc| { res + per_relay_parent From fe3c09d42a2f6996a9e6c36a9e8f4ca12fdcf696 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Wed, 11 Sep 2024 17:46:11 +0300 Subject: [PATCH 41/55] Rework pre-prospective parachains tests to work with claim queue --- .../src/validator_side/tests/mod.rs | 447 +++++------------- 1 file changed, 112 insertions(+), 335 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index cc097ba1065c..6da914028351 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use self::prospective_parachains::update_view; + use super::*; use assert_matches::assert_matches; use futures::{executor, future, Future}; @@ -29,15 +31,13 @@ use std::{ }; use polkadot_node_network_protocol::{ - our_view, peer_set::CollationVersion, request_response::{Requests, ResponseSender}, ObservedRole, }; use polkadot_node_primitives::{BlockData, PoV}; -use polkadot_node_subsystem::{ - errors::RuntimeApiError, - messages::{AllMessages, ReportPeerMessage, RuntimeApiMessage, RuntimeApiRequest}, +use polkadot_node_subsystem::messages::{ + AllMessages, ReportPeerMessage, RuntimeApiMessage, RuntimeApiRequest, }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_node_subsystem_util::{reputation::add_reputation, TimeoutExt}; @@ -50,16 +50,13 @@ use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt_bad_sig, dummy_hash, }; -mod collation; +// mod collation; mod prospective_parachains; const ACTIVITY_TIMEOUT: Duration = Duration::from_millis(500); const DECLARE_TIMEOUT: Duration = Duration::from_millis(25); const REPUTATION_CHANGE_TEST_INTERVAL: Duration = Duration::from_millis(10); -const ASYNC_BACKING_DISABLED_ERROR: RuntimeApiError = - RuntimeApiError::NotSupported { runtime_api_name: "test-runtime" }; - fn dummy_pvd() -> PersistedValidationData { PersistedValidationData { parent_head: HeadData(vec![7, 8, 9]), @@ -213,6 +210,41 @@ impl TestState { state } + + fn with_one_scheduled_para() -> Self { + let mut state = Self::default(); + + let cores = vec![CoreState::Scheduled(ScheduledCore { + para_id: ParaId::from(Self::CHAIN_IDS[0]), + collator: None, + })]; + + let validator_groups = vec![vec![ValidatorIndex(0), ValidatorIndex(1)]]; + + let mut claim_queue = BTreeMap::new(); + claim_queue.insert( + CoreIndex(0), + VecDeque::from_iter( + [ + ParaId::from(Self::CHAIN_IDS[0]), + ParaId::from(Self::CHAIN_IDS[0]), + ParaId::from(Self::CHAIN_IDS[0]), + ] + .into_iter(), + ), + ); + + assert!( + claim_queue.get(&CoreIndex(0)).unwrap().len() == + Self::ASYNC_BACKING_PARAMS.allowed_ancestry_len as usize + ); + + state.cores = cores; + state.validator_groups = validator_groups; + state.claim_queue = Some(claim_queue); + + state + } } type VirtualOverseer = @@ -307,73 +339,6 @@ async fn overseer_signal(overseer: &mut VirtualOverseer, signal: OverseerSignal) .expect(&format!("{:?} is more than enough for sending signals.", TIMEOUT)); } -async fn respond_to_core_info_queries( - virtual_overseer: &mut VirtualOverseer, - test_state: &TestState, -) { - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::Validators(tx), - )) => { - let _ = tx.send(Ok(test_state.validator_public.clone())); - } - ); - - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::ValidatorGroups(tx), - )) => { - let _ = tx.send(Ok(( - test_state.validator_groups.clone(), - test_state.group_rotation_info.clone(), - ))); - } - ); - - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::AvailabilityCores(tx), - )) => { - let _ = tx.send(Ok(test_state.cores.clone())); - } - ); - - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::Version(tx), - )) => { - let _ = tx.send(Ok(RuntimeApiRequest::CLAIM_QUEUE_RUNTIME_REQUIREMENT)); - } - ); - - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - _, - RuntimeApiRequest::ClaimQueue(tx), - )) => { - match test_state.claim_queue { - Some(ref claim_queue) => { - let _ = tx.send(Ok(claim_queue.clone())); - }, - None => { - let _ = tx.send(Err(RuntimeApiError::NotSupported { runtime_api_name: "doesnt_matter" })); - } - - } - - } - ); -} - /// Assert that the next message is a `CandidateBacking(Second())`. async fn assert_candidate_backing_second( virtual_overseer: &mut VirtualOverseer, @@ -549,148 +514,6 @@ async fn advertise_collation( .await; } -async fn assert_async_backing_params_request(virtual_overseer: &mut VirtualOverseer, hash: Hash) { - assert_matches!( - overseer_recv(virtual_overseer).await, - AllMessages::RuntimeApi(RuntimeApiMessage::Request( - relay_parent, - RuntimeApiRequest::AsyncBackingParams(tx) - )) => { - assert_eq!(relay_parent, hash); - tx.send(Err(ASYNC_BACKING_DISABLED_ERROR)).unwrap(); - } - ); -} - -// As we receive a relevant advertisement act on it and issue a collation request. -#[test] -fn act_on_advertisement() { - let test_state = TestState::default(); - - test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { - let TestHarness { mut virtual_overseer, .. } = test_harness; - - let pair = CollatorPair::generate().0; - gum::trace!("activating"); - - overseer_send( - &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), - ) - .await; - - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - - let peer_b = PeerId::random(); - - connect_and_declare_collator( - &mut virtual_overseer, - peer_b, - pair.clone(), - test_state.chain_ids[0], - CollationVersion::V1, - ) - .await; - - advertise_collation(&mut virtual_overseer, peer_b, test_state.relay_parent, None).await; - - assert_fetch_collation_request( - &mut virtual_overseer, - test_state.relay_parent, - test_state.chain_ids[0], - None, - ) - .await; - - virtual_overseer - }); -} - -/// Tests that validator side works with v2 network protocol -/// before async backing is enabled. -#[test] -fn act_on_advertisement_v2() { - let test_state = TestState::default(); - - test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { - let TestHarness { mut virtual_overseer, .. } = test_harness; - - let pair = CollatorPair::generate().0; - gum::trace!("activating"); - - overseer_send( - &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), - ) - .await; - - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - - let peer_b = PeerId::random(); - - connect_and_declare_collator( - &mut virtual_overseer, - peer_b, - pair.clone(), - test_state.chain_ids[0], - CollationVersion::V2, - ) - .await; - - let pov = PoV { block_data: BlockData(vec![]) }; - let mut candidate_a = - dummy_candidate_receipt_bad_sig(dummy_hash(), Some(Default::default())); - candidate_a.descriptor.para_id = test_state.chain_ids[0]; - candidate_a.descriptor.relay_parent = test_state.relay_parent; - candidate_a.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); - - let candidate_hash = candidate_a.hash(); - let parent_head_data_hash = Hash::zero(); - // v2 advertisement. - advertise_collation( - &mut virtual_overseer, - peer_b, - test_state.relay_parent, - Some((candidate_hash, parent_head_data_hash)), - ) - .await; - - let response_channel = assert_fetch_collation_request( - &mut virtual_overseer, - test_state.relay_parent, - test_state.chain_ids[0], - Some(candidate_hash), - ) - .await; - - response_channel - .send(Ok(( - request_v1::CollationFetchingResponse::Collation(candidate_a.clone(), pov.clone()) - .encode(), - ProtocolName::from(""), - ))) - .expect("Sending response should succeed"); - - assert_candidate_backing_second( - &mut virtual_overseer, - test_state.relay_parent, - test_state.chain_ids[0], - &pov, - // Async backing isn't enabled and thus it should do it the old way. - CollationVersion::V1, - ) - .await; - - virtual_overseer - }); -} - // Test that other subsystems may modify collators' reputations. #[test] fn collator_reporting_works() { @@ -699,18 +522,18 @@ fn collator_reporting_works() { test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, .. } = test_harness; - overseer_send( + let head = Hash::from_low_u64_be(128); + let head_num: u32 = 0; + + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), + &test_state, + vec![(head, head_num)], + 1, + &test_state.async_backing_params, ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - let peer_b = PeerId::random(); let peer_c = PeerId::random(); @@ -812,22 +635,15 @@ fn fetch_one_collation_at_a_time() { let TestHarness { mut virtual_overseer, .. } = test_harness; let second = Hash::random(); - let our_view = our_view![test_state.relay_parent, second]; - - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view.clone(), - )), + &test_state, + vec![(test_state.relay_parent, 0), (second, 1)], + 2, + &test_state.async_backing_params, ) .await; - // Iter over view since the order may change due to sorted invariant. - for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - } - let peer_b = PeerId::random(); let peer_c = PeerId::random(); @@ -902,28 +718,22 @@ fn fetch_one_collation_at_a_time() { /// timeout and in case of an error. #[test] fn fetches_next_collation() { - let test_state = TestState::default(); + let test_state = TestState::with_one_scheduled_para(); test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, .. } = test_harness; let second = Hash::random(); - let our_view = our_view![test_state.relay_parent, second]; - - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view.clone(), - )), + &test_state, + vec![(test_state.relay_parent, 0), (second, 1)], + 2, + &test_state.async_backing_params, ) .await; - for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - } - let peer_b = PeerId::random(); let peer_c = PeerId::random(); let peer_d = PeerId::random(); @@ -1031,17 +841,15 @@ fn reject_connection_to_next_group() { test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, .. } = test_harness; - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), + &test_state, + vec![(test_state.relay_parent, 0)], + 1, + &test_state.async_backing_params, ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - let peer_b = PeerId::random(); connect_and_declare_collator( @@ -1080,21 +888,15 @@ fn fetch_next_collation_on_invalid_collation() { let second = Hash::random(); - let our_view = our_view![test_state.relay_parent, second]; - - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view.clone(), - )), + &test_state, + vec![(test_state.relay_parent, 0), (second, 1)], + 2, + &test_state.async_backing_params, ) .await; - for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - } - let peer_b = PeerId::random(); let peer_c = PeerId::random(); @@ -1189,19 +991,15 @@ fn inactive_disconnected() { let pair = CollatorPair::generate().0; - let hash_a = test_state.relay_parent; - - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![hash_a], - )), + &test_state, + vec![(test_state.relay_parent, 0)], + 1, + &test_state.async_backing_params, ) .await; - assert_async_backing_params_request(&mut virtual_overseer, hash_a).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - let peer_b = PeerId::random(); connect_and_declare_collator( @@ -1231,7 +1029,7 @@ fn inactive_disconnected() { #[test] fn activity_extends_life() { - let test_state = TestState::default(); + let test_state = TestState::with_one_scheduled_para(); test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, .. } = test_harness; @@ -1242,21 +1040,15 @@ fn activity_extends_life() { let hash_b = Hash::repeat_byte(1); let hash_c = Hash::repeat_byte(2); - let our_view = our_view![hash_a, hash_b, hash_c]; - - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view.clone(), - )), + &test_state, + vec![(hash_a, 0), (hash_b, 1), (hash_c, 1)], + 3, + &test_state.async_backing_params, ) .await; - for hash in our_view.iter() { - assert_async_backing_params_request(&mut virtual_overseer, *hash).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - } - let peer_b = PeerId::random(); connect_and_declare_collator( @@ -1319,17 +1111,15 @@ fn disconnect_if_no_declare() { test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, .. } = test_harness; - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), + &test_state, + vec![(test_state.relay_parent, 0)], + 1, + &test_state.async_backing_params, ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - let peer_b = PeerId::random(); overseer_send( @@ -1355,22 +1145,18 @@ fn disconnect_if_wrong_declare() { test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, .. } = test_harness; - let pair = CollatorPair::generate().0; + let peer_b = PeerId::random(); - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), + &test_state, + vec![(test_state.relay_parent, 0)], + 1, + &test_state.async_backing_params, ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - - let peer_b = PeerId::random(); - overseer_send( &mut virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( @@ -1417,22 +1203,18 @@ fn delay_reputation_change() { test_harness(ReputationAggregator::new(|_| false), |test_harness| async move { let TestHarness { mut virtual_overseer, .. } = test_harness; - let pair = CollatorPair::generate().0; + let peer_b = PeerId::random(); - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), + &test_state, + vec![(test_state.relay_parent, 0)], + 1, + &test_state.async_backing_params, ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - - let peer_b = PeerId::random(); - overseer_send( &mut virtual_overseer, CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::PeerConnected( @@ -1506,43 +1288,38 @@ fn view_change_clears_old_collators() { let pair = CollatorPair::generate().0; - overseer_send( + let peer = PeerId::random(); + + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![test_state.relay_parent], - )), + &test_state, + vec![(test_state.relay_parent, 0)], + 1, + &test_state.async_backing_params, ) .await; - assert_async_backing_params_request(&mut virtual_overseer, test_state.relay_parent).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - - let peer_b = PeerId::random(); - connect_and_declare_collator( &mut virtual_overseer, - peer_b, + peer, pair.clone(), test_state.chain_ids[0], CollationVersion::V1, ) .await; - let hash_b = Hash::repeat_byte(69); + test_state.group_rotation_info = test_state.group_rotation_info.bump_rotation(); - overseer_send( + update_view( &mut virtual_overseer, - CollatorProtocolMessage::NetworkBridgeUpdate(NetworkBridgeEvent::OurViewChange( - our_view![hash_b], - )), + &test_state, + vec![], + 0, + &test_state.async_backing_params, ) .await; - test_state.group_rotation_info = test_state.group_rotation_info.bump_rotation(); - assert_async_backing_params_request(&mut virtual_overseer, hash_b).await; - respond_to_core_info_queries(&mut virtual_overseer, &test_state).await; - - assert_collator_disconnect(&mut virtual_overseer, peer_b).await; + assert_collator_disconnect(&mut virtual_overseer, peer).await; virtual_overseer }) From b9ab5794b9c537eaac5daaa4962f9d4d79fe2b0a Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 12 Sep 2024 16:17:53 +0300 Subject: [PATCH 42/55] Fix `collation_fetches_without_claimqueue` --- .../tests/prospective_parachains.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 64b392dda4f1..04b26d0979e0 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -1597,6 +1597,8 @@ fn fair_collation_fetches() { }); } +// This should not happen in practice since claim queue is supported on all networks but just in +// case validate that the fallback works as expected #[test] fn collation_fetches_without_claimqueue() { let test_state = TestState::without_claim_queue(); @@ -1659,18 +1661,16 @@ fn collation_fetches_without_claimqueue() { } ); - // in fallback mode up to `max_candidate_depth` collations are accepted - for i in 0..test_state.async_backing_params.max_candidate_depth + 1 { - submit_second_and_assert( - &mut virtual_overseer, - keystore.clone(), - ParaId::from(TestState::CHAIN_IDS[0]), - head_b, - peer_a, - HeadData(vec![i as u8]), - ) - .await; - } + // in fallback mode we only accept what's scheduled on the core + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + ParaId::from(TestState::CHAIN_IDS[0]), + head_b, + peer_a, + HeadData(vec![0 as u8]), + ) + .await; // `peer_a` sends another advertisement and it is ignored let candidate_hash = CandidateHash(Hash::repeat_byte(0xAA)); From fe623bcd6f1d83db9dbf8f11e495f74fc94d4747 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Fri, 13 Sep 2024 15:17:55 +0300 Subject: [PATCH 43/55] Test - `collation_fetching_prefer_entries_earlier_in_claim_queue` --- .../tests/prospective_parachains.rs | 285 ++++++++++++++++-- 1 file changed, 261 insertions(+), 24 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 04b26d0979e0..6f4dc61b4742 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -334,6 +334,7 @@ async fn assert_persisted_validation_data( } } +// Combines dummy candidate creation, advertisement and fetching in a single call async fn submit_second_and_assert( virtual_overseer: &mut VirtualOverseer, keystore: KeystorePtr, @@ -342,6 +343,50 @@ async fn submit_second_and_assert( collator: PeerId, candidate_head_data: HeadData, ) { + let (candidate, commitments) = + create_dummy_candidate_and_commitments(para_id, candidate_head_data, relay_parent); + + let candidate_hash = candidate.hash(); + let parent_head_data_hash = Hash::zero(); + + assert_advertise_collation( + virtual_overseer, + collator, + relay_parent, + para_id, + (candidate_hash, parent_head_data_hash), + ) + .await; + + let response_channel = assert_fetch_collation_request( + virtual_overseer, + relay_parent, + para_id, + Some(candidate_hash), + ) + .await; + + let pov = PoV { block_data: BlockData(vec![1]) }; + + send_collation_and_assert_processing( + virtual_overseer, + keystore, + relay_parent, + para_id, + collator, + response_channel, + candidate, + commitments, + pov, + ) + .await; +} + +fn create_dummy_candidate_and_commitments( + para_id: ParaId, + candidate_head_data: HeadData, + relay_parent: Hash, +) -> (CandidateReceipt, CandidateCommitments) { let mut candidate = dummy_candidate_receipt_bad_sig(relay_parent, Some(Default::default())); candidate.descriptor.para_id = para_id; candidate.descriptor.persisted_validation_data_hash = dummy_pvd().hash(); @@ -355,38 +400,41 @@ async fn submit_second_and_assert( }; candidate.commitments_hash = commitments.hash(); - let candidate_hash = candidate.hash(); - let parent_head_data_hash = Hash::zero(); + (candidate, commitments) +} - advertise_collation( - virtual_overseer, - collator, - relay_parent, - Some((candidate_hash, parent_head_data_hash)), - ) - .await; +async fn assert_advertise_collation( + virtual_overseer: &mut VirtualOverseer, + peer: PeerId, + relay_parent: Hash, + expected_para_id: ParaId, + candidate: (CandidateHash, Hash), +) { + advertise_collation(virtual_overseer, peer, relay_parent, Some(candidate)).await; assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::CandidateBacking( CandidateBackingMessage::CanSecond(request, tx), ) => { - assert_eq!(request.candidate_hash, candidate_hash); - assert_eq!(request.candidate_para_id, para_id); - assert_eq!(request.parent_head_data_hash, parent_head_data_hash); + assert_eq!(request.candidate_hash, candidate.0); + assert_eq!(request.candidate_para_id, expected_para_id); + assert_eq!(request.parent_head_data_hash, candidate.1); tx.send(true).expect("receiving side should be alive"); } ); +} - let response_channel = assert_fetch_collation_request( - virtual_overseer, - relay_parent, - para_id, - Some(candidate_hash), - ) - .await; - - let pov = PoV { block_data: BlockData(vec![1]) }; - +async fn send_collation_and_assert_processing( + virtual_overseer: &mut VirtualOverseer, + keystore: KeystorePtr, + relay_parent: Hash, + expected_para_id: ParaId, + expected_peer_id: PeerId, + response_channel: ResponseSender, + candidate: CandidateReceipt, + commitments: CandidateCommitments, + pov: PoV, +) { response_channel .send(Ok(( request_v2::CollationFetchingResponse::Collation(candidate.clone(), pov.clone()) @@ -398,7 +446,7 @@ async fn submit_second_and_assert( assert_candidate_backing_second( virtual_overseer, relay_parent, - para_id, + expected_para_id, &pov, CollationVersion::V2, ) @@ -408,7 +456,13 @@ async fn submit_second_and_assert( send_seconded_statement(virtual_overseer, keystore.clone(), &candidate).await; - assert_collation_seconded(virtual_overseer, relay_parent, collator, CollationVersion::V2).await; + assert_collation_seconded( + virtual_overseer, + relay_parent, + expected_peer_id, + CollationVersion::V2, + ) + .await; } #[test] @@ -1687,3 +1741,186 @@ fn collation_fetches_without_claimqueue() { virtual_overseer }); } + +#[test] +fn collation_fetching_prefer_entries_earlier_in_claim_queue() { + let test_state = TestState::with_shared_core(); + + test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { + let TestHarness { mut virtual_overseer, keystore } = test_harness; + + let pair_a = CollatorPair::generate().0; + let collator_a = PeerId::random(); + let para_id_a = test_state.chain_ids[0]; + + let pair_b = CollatorPair::generate().0; + let collator_b = PeerId::random(); + let para_id_b = test_state.chain_ids[1]; + + let head = Hash::from_low_u64_be(128); + let head_num: u32 = 2; + + update_view( + &mut virtual_overseer, + &test_state, + vec![(head, head_num)], + 1, + &test_state.async_backing_params, + ) + .await; + + connect_and_declare_collator( + &mut virtual_overseer, + collator_a, + pair_a.clone(), + para_id_a, + CollationVersion::V2, + ) + .await; + + connect_and_declare_collator( + &mut virtual_overseer, + collator_b, + pair_b.clone(), + para_id_b, + CollationVersion::V2, + ) + .await; + + let (candidate_a1, commitments_a1) = + create_dummy_candidate_and_commitments(para_id_a, HeadData(vec![0 as u8]), head); + let (candidate_b1, commitments_b1) = + create_dummy_candidate_and_commitments(para_id_b, HeadData(vec![1 as u8]), head); + let (candidate_a2, commitments_a2) = + create_dummy_candidate_and_commitments(para_id_a, HeadData(vec![2 as u8]), head); + let (candidate_a3, _) = + create_dummy_candidate_and_commitments(para_id_a, HeadData(vec![3 as u8]), head); + let parent_head_data_a1 = HeadData(vec![0 as u8]); + let parent_head_data_b1 = HeadData(vec![1 as u8]); + let parent_head_data_a2 = HeadData(vec![2 as u8]); + let parent_head_data_a3 = HeadData(vec![3 as u8]); + + // advertise a collation for `para_id_a` but don't send the collation. This will be a + // pending fetch. + assert_advertise_collation( + &mut virtual_overseer, + collator_a, + head, + para_id_a, + (candidate_a1.hash(), parent_head_data_a1.hash()), + ) + .await; + + let response_channel_a1 = assert_fetch_collation_request( + &mut virtual_overseer, + head, + para_id_a, + Some(candidate_a1.hash()), + ) + .await; + // + + // advertise another collation for `para_id_a`. This one should be fetched last. + assert_advertise_collation( + &mut virtual_overseer, + collator_a, + head, + para_id_a, + (candidate_a2.hash(), parent_head_data_a2.hash()), + ) + .await; + + // There is a pending collation so nothing should be fetched + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + // Advertise a collation for `para_id_b`. This should be fetched second + assert_advertise_collation( + &mut virtual_overseer, + collator_b, + head, + para_id_b, + (candidate_b1.hash(), parent_head_data_b1.hash()), + ) + .await; + + // Again - no fetch because of the pending collation + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + //Now send a response for the first fetch and examine the second fetch + send_collation_and_assert_processing( + &mut virtual_overseer, + keystore.clone(), + head, + para_id_a, + collator_a, + response_channel_a1, + candidate_a1, + commitments_a1, + PoV { block_data: BlockData(vec![1]) }, + ) + .await; + + // The next fetch should be for `para_id_b` + let response_channel_b = assert_fetch_collation_request( + &mut virtual_overseer, + head, + para_id_b, + Some(candidate_b1.hash()), + ) + .await; + + send_collation_and_assert_processing( + &mut virtual_overseer, + keystore.clone(), + head, + para_id_b, + collator_b, + response_channel_b, + candidate_b1, + commitments_b1, + PoV { block_data: BlockData(vec![2]) }, + ) + .await; + + // and the final one for `para_id_a` + let response_channel_a2 = assert_fetch_collation_request( + &mut virtual_overseer, + head, + para_id_a, + Some(candidate_a2.hash()), + ) + .await; + + // Advertise another collation for `para_id_a`. This should be rejected as there is no slot + // in the claim queue for it. One is fetched and one is pending. + advertise_collation( + &mut virtual_overseer, + collator_a, + head, + Some((candidate_a3.hash(), parent_head_data_a3.hash())), + ) + .await; + + // `CanSecond` shouldn't be sent as the advertisement should be ignored + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + // Fetch the pending collation + send_collation_and_assert_processing( + &mut virtual_overseer, + keystore.clone(), + head, + para_id_a, + collator_a, + response_channel_a2, + candidate_a2, + commitments_a2, + PoV { block_data: BlockData(vec![3]) }, + ) + .await; + + virtual_overseer + }); +} From d21668914f53b1435424e44ad3e73c4b9799592c Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Fri, 13 Sep 2024 16:24:40 +0300 Subject: [PATCH 44/55] Remove collations test file - all tests are moved in prospective_parachains --- .../src/validator_side/tests/collation.rs | 627 ------------------ .../src/validator_side/tests/mod.rs | 1 - 2 files changed, 628 deletions(-) delete mode 100644 polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs deleted file mode 100644 index 24972e66926d..000000000000 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/collation.rs +++ /dev/null @@ -1,627 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -use polkadot_primitives::{CandidateHash, CollatorId, Hash, Id as ParaId}; - -use sc_network::PeerId; -use sp_core::sr25519; - -use crate::validator_side::tests::CollationStatus; - -use super::{Collations, PendingCollation, ProspectiveCandidate}; - -#[test] -fn cant_add_more_than_claim_queue() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let para_b = ParaId::from(2); - let assignments = vec![para_a, para_b, para_a]; - let max_candidate_depth = 4; - let claim_queue_support = true; - - let mut collations = Collations::new(&assignments, claim_queue_support); - - // first collation for `para_a` is in the limit - assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_a)); - collations.note_seconded(para_a); - // and `para_b` is not affected - assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); - - // second collation for `para_a` is also in the limit - assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_a)); - collations.note_seconded(para_a); - - // `para_b`` is still not affected - assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); - - // third collation for `para_a`` will be above the limit - assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_a)); - - // one fetch for b - assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); - collations.note_seconded(para_b); - - // and now both paras are over limit - assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_a)); - assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_b)); -} - -#[test] -fn pending_fetches_are_counted() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let para_b = ParaId::from(2); - let assignments = vec![para_a, para_b, para_a]; - let max_candidate_depth = 4; - let claim_queue_support = true; - - let mut collations = Collations::new(&assignments, claim_queue_support); - collations.status = CollationStatus::Fetching(para_a); //para_a is pending - - // first collation for `para_a` is in the limit - assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_a)); - collations.note_seconded(para_a); - - // second collation for `para_a` is not in the limit due to the pending fetch - assert!(collations.is_seconded_limit_reached(max_candidate_depth, para_a)); - - // a collation for `para_b` is accepted since the pending fetch is for `para_a` - assert!(!collations.is_seconded_limit_reached(max_candidate_depth, para_b)); -} - -#[test] -fn collation_fetching_respects_claim_queue() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); - let peer_a = PeerId::random(); - - let para_b = ParaId::from(2); - let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); - let peer_b = PeerId::random(); - - let claim_queue = vec![para_a, para_b, para_a]; - let claim_queue_support = true; - - let mut collations = Collations::new(&claim_queue, claim_queue_support); - - collations.fetching_from = None; - collations.status = CollationStatus::Waiting; //nothing pending - - let relay_parent = Hash::repeat_byte(0x01); - - let collation_a1 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(1)), - parent_head_data_hash: Hash::repeat_byte(1), - }), - ), - collator_id_a.clone(), - ); - - let collation_a2 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(2)), - parent_head_data_hash: Hash::repeat_byte(2), - }), - ), - collator_id_a.clone(), - ); - - let collation_b1 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(3)), - parent_head_data_hash: Hash::repeat_byte(3), - }), - ), - collator_id_b.clone(), - ); - - collations.add_to_waiting_queue(collation_b1.clone()); - collations.add_to_waiting_queue(collation_a1.clone()); - collations.add_to_waiting_queue(collation_a2.clone()); - - assert_eq!( - Some(collation_a1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a1.0.para_id); - - assert_eq!( - Some(collation_b1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_b1.0.para_id); - - assert_eq!( - Some(collation_a2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a2.0.para_id); -} - -#[test] -fn collation_fetching_fallback_works() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); - let peer_a = PeerId::random(); - - let claim_queue = vec![para_a]; - let claim_queue_support = false; - - let mut collations = Collations::new(&claim_queue, claim_queue_support); - - collations.fetching_from = None; - collations.status = CollationStatus::Waiting; //nothing pending - - let relay_parent = Hash::repeat_byte(0x01); - - let collation_a1 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(1)), - parent_head_data_hash: Hash::repeat_byte(1), - }), - ), - collator_id_a.clone(), - ); - - let collation_a2 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(2)), - parent_head_data_hash: Hash::repeat_byte(2), - }), - ), - collator_id_a.clone(), - ); - - // Collations will be fetched in the order they were added - collations.add_to_waiting_queue(collation_a1.clone()); - collations.add_to_waiting_queue(collation_a2.clone()); - - assert_eq!( - Some(collation_a1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a1.0.para_id); - - assert_eq!( - Some(collation_a2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a2.0.para_id); -} - -#[test] -fn collation_fetching_prefer_entries_earlier_in_claim_queue() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); - let peer_a = PeerId::random(); - - let para_b = ParaId::from(2); - let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); - let peer_b = PeerId::random(); - - let para_c = ParaId::from(3); - let collator_id_c = CollatorId::from(sr25519::Public::from_raw([30u8; 32])); - let peer_c = PeerId::random(); - - let claim_queue = vec![para_a, para_b, para_a, para_b, para_c, para_c]; - let claim_queue_support = true; - - let mut collations = Collations::new(&claim_queue, claim_queue_support); - collations.fetching_from = None; - collations.status = CollationStatus::Waiting; //nothing pending - - let relay_parent = Hash::repeat_byte(0x01); - - let collation_a1 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(1)), - parent_head_data_hash: Hash::repeat_byte(1), - }), - ), - collator_id_a.clone(), - ); - - let collation_a2 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(2)), - parent_head_data_hash: Hash::repeat_byte(2), - }), - ), - collator_id_a.clone(), - ); - - let collation_b1 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(3)), - parent_head_data_hash: Hash::repeat_byte(3), - }), - ), - collator_id_b.clone(), - ); - - let collation_b2 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(4)), - parent_head_data_hash: Hash::repeat_byte(4), - }), - ), - collator_id_b.clone(), - ); - - let collation_c1 = ( - PendingCollation::new( - relay_parent, - para_c, - &peer_c, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(5)), - parent_head_data_hash: Hash::repeat_byte(5), - }), - ), - collator_id_c.clone(), - ); - - let collation_c2 = ( - PendingCollation::new( - relay_parent, - para_c, - &peer_c, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(6)), - parent_head_data_hash: Hash::repeat_byte(6), - }), - ), - collator_id_c.clone(), - ); - - // Despite the order here the fetches should follow the claim queue - collations.add_to_waiting_queue(collation_c1.clone()); - collations.add_to_waiting_queue(collation_c2.clone()); - collations.add_to_waiting_queue(collation_b1.clone()); - collations.add_to_waiting_queue(collation_b2.clone()); - collations.add_to_waiting_queue(collation_a1.clone()); - collations.add_to_waiting_queue(collation_a2.clone()); - - assert_eq!( - Some(collation_a1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a1.0.para_id); - - assert_eq!( - Some(collation_b1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_b1.0.para_id); - - assert_eq!( - Some(collation_a2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a2.0.para_id); - - assert_eq!( - Some(collation_b2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_b2.0.para_id); - - assert_eq!( - Some(collation_c1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_c1.0.para_id); - - assert_eq!( - Some(collation_c2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_c2.0.para_id); -} - -#[test] -fn collation_fetching_fills_holes_in_claim_queue() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); - let peer_a = PeerId::random(); - - let para_b = ParaId::from(2); - let collator_id_b = CollatorId::from(sr25519::Public::from_raw([20u8; 32])); - let peer_b = PeerId::random(); - - let para_c = ParaId::from(3); - let collator_id_c = CollatorId::from(sr25519::Public::from_raw([30u8; 32])); - let peer_c = PeerId::random(); - - let claim_queue = vec![para_a, para_b, para_a, para_b, para_c, para_c]; - let claim_queue_support = true; - - let mut collations = Collations::new(&claim_queue, claim_queue_support); - collations.fetching_from = None; - collations.status = CollationStatus::Waiting; //nothing pending - - let relay_parent = Hash::repeat_byte(0x01); - - let collation_a1 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(1)), - parent_head_data_hash: Hash::repeat_byte(1), - }), - ), - collator_id_a.clone(), - ); - - let collation_a2 = ( - PendingCollation::new( - relay_parent, - para_a, - &peer_a, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(2)), - parent_head_data_hash: Hash::repeat_byte(2), - }), - ), - collator_id_a.clone(), - ); - - let collation_b1 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(3)), - parent_head_data_hash: Hash::repeat_byte(3), - }), - ), - collator_id_b.clone(), - ); - - let collation_b2 = ( - PendingCollation::new( - relay_parent, - para_b, - &peer_b, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(4)), - parent_head_data_hash: Hash::repeat_byte(4), - }), - ), - collator_id_b.clone(), - ); - - let collation_c1 = ( - PendingCollation::new( - relay_parent, - para_c, - &peer_c, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(5)), - parent_head_data_hash: Hash::repeat_byte(5), - }), - ), - collator_id_c.clone(), - ); - - let collation_c2 = ( - PendingCollation::new( - relay_parent, - para_c, - &peer_c, - Some(ProspectiveCandidate { - candidate_hash: CandidateHash(Hash::repeat_byte(6)), - parent_head_data_hash: Hash::repeat_byte(6), - }), - ), - collator_id_c.clone(), - ); - - collations.add_to_waiting_queue(collation_c1.clone()); - collations.add_to_waiting_queue(collation_a1.clone()); - - assert_eq!( - Some(collation_a1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a1.0.para_id); - - // fetch c1 since there is nothing better to fetch - assert_eq!( - Some(collation_c1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_c1.0.para_id); - - // b1 should be prioritized since there is a hole in the claim queue - collations.add_to_waiting_queue(collation_c2.clone()); - collations.add_to_waiting_queue(collation_b1.clone()); - - assert_eq!( - Some(collation_b1.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_b1.0.para_id); - - assert_eq!( - Some(collation_c2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_c2.0.para_id); - - // same with a2 - collations.add_to_waiting_queue(collation_b2.clone()); - collations.add_to_waiting_queue(collation_a2.clone()); - - assert_eq!( - Some(collation_a2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_a2.0.para_id); - - assert_eq!( - Some(collation_b2.clone()), - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); - collations.note_seconded(collation_b2.0.para_id); -} - -#[test] -fn collations_fetching_respects_seconded_limit() { - sp_tracing::init_for_tests(); - - let para_a = ParaId::from(1); - let collator_id_a = CollatorId::from(sr25519::Public::from_raw([10u8; 32])); - - let para_b = ParaId::from(2); - - let claim_queue = vec![para_a, para_b, para_a]; - let claim_queue_support = true; - - let mut collations = Collations::new(&claim_queue, claim_queue_support); - collations.fetching_from = None; - collations.status = CollationStatus::Fetching(para_a); //para_a is pending - - collations.note_seconded(para_a); - collations.note_seconded(para_a); - - assert_eq!( - None, - collations.get_next_collation_to_fetch( - // doesn't matter since `fetching_from` is `None` - &(collator_id_a.clone(), Some(CandidateHash(Hash::repeat_byte(0)))), - &claim_queue, - ) - ); -} diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs index 6da914028351..8786b845a6b1 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/mod.rs @@ -50,7 +50,6 @@ use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt_bad_sig, dummy_hash, }; -// mod collation; mod prospective_parachains; const ACTIVITY_TIMEOUT: Duration = Duration::from_millis(500); From ea99c7aeded6c8b7b6f5bb4f5b380fd6742ab278 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 16 Sep 2024 10:03:45 +0300 Subject: [PATCH 45/55] fixup - collation_fetching_prefer_entries_earlier_in_claim_queue --- .../src/validator_side/tests/prospective_parachains.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 6f4dc61b4742..1c60a0b15215 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -1818,7 +1818,6 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { Some(candidate_a1.hash()), ) .await; - // // advertise another collation for `para_id_a`. This one should be fetched last. assert_advertise_collation( From ee155f54fa64ac38b9ce9310213dda4f086bbee6 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Mon, 16 Sep 2024 13:22:50 +0300 Subject: [PATCH 46/55] New test - `collation_fetching_considers_advertisements_from_the_whole_view` --- .../tests/prospective_parachains.rs | 158 ++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 1c60a0b15215..8bce4e959915 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -1923,3 +1923,161 @@ fn collation_fetching_prefer_entries_earlier_in_claim_queue() { virtual_overseer }); } + +#[test] +fn collation_fetching_considers_advertisements_from_the_whole_view() { + let test_state = TestState::with_shared_core(); + + test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { + let TestHarness { mut virtual_overseer, keystore } = test_harness; + + let pair_a = CollatorPair::generate().0; + let collator_a = PeerId::random(); + let para_id_a = test_state.chain_ids[0]; + + let pair_b = CollatorPair::generate().0; + let collator_b = PeerId::random(); + let para_id_b = test_state.chain_ids[1]; + + let relay_parent_2 = Hash::from_low_u64_be(128); + + update_view( + &mut virtual_overseer, + &test_state, + vec![(relay_parent_2, 2)], + 1, + &test_state.async_backing_params, + ) + .await; + + connect_and_declare_collator( + &mut virtual_overseer, + collator_a, + pair_a.clone(), + para_id_a, + CollationVersion::V2, + ) + .await; + + connect_and_declare_collator( + &mut virtual_overseer, + collator_b, + pair_b.clone(), + para_id_b, + CollationVersion::V2, + ) + .await; + + // Two advertisements for `para_id_a` at `relay_parent_2` + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + para_id_a, + relay_parent_2, + collator_a, + HeadData(vec![0 as u8]), + ) + .await; + + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + para_id_a, + relay_parent_2, + collator_a, + HeadData(vec![1 as u8]), + ) + .await; + + // parent hashes are hardcoded in `get_parent_hash` (called from `update_view`) to be + // `current hash + 1` so we need to craft them carefully (decrement by 2) in order to make + // them fall in the same view. + let relay_parent_4 = Hash::from_low_u64_be(126); + + update_view( + &mut virtual_overseer, + &test_state, + vec![(relay_parent_4, 4)], + 1, + &test_state.async_backing_params, + ) + .await; + + // One advertisement for `para_id_b` at `relay_parent_4` + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + para_id_b, + relay_parent_4, + collator_b, + HeadData(vec![3 as u8]), + ) + .await; + + // At this point the claim queue is satisfied and any advertisement at `relay_parent_4` + // must be ignored + + // Advertisement for `para_id_a` at `relay_parent_4` which must be ignored + let (candidate_a, _) = create_dummy_candidate_and_commitments( + para_id_a, + HeadData(vec![5 as u8]), + relay_parent_4, + ); + let parent_head_data_a = HeadData(vec![5 as u8]); + + advertise_collation( + &mut virtual_overseer, + collator_a, + relay_parent_4, + Some((candidate_a.hash(), parent_head_data_a.hash())), + ) + .await; + + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + // Advertisement for `para_id_b` at `relay_parent_4` which must be ignored + let (candidate_b, _) = create_dummy_candidate_and_commitments( + para_id_b, + HeadData(vec![6 as u8]), + relay_parent_4, + ); + let parent_head_data_b = HeadData(vec![6 as u8]); + + advertise_collation( + &mut virtual_overseer, + collator_b, + relay_parent_4, + Some((candidate_b.hash(), parent_head_data_b.hash())), + ) + .await; + + // `CanSecond` shouldn't be sent as the advertisement should be ignored + test_helpers::Yield::new().await; + assert_matches!(virtual_overseer.recv().now_or_never(), None); + + // At `relay_parent_6` the advertisement for `para_id_b` falls out of the view so a new one + // can be accepted + let relay_parent_6 = Hash::from_low_u64_be(124); + update_view( + &mut virtual_overseer, + &test_state, + vec![(relay_parent_6, 6)], + 1, + &test_state.async_backing_params, + ) + .await; + + submit_second_and_assert( + &mut virtual_overseer, + keystore.clone(), + para_id_a, + relay_parent_6, + collator_a, + HeadData(vec![3 as u8]), + ) + .await; + + virtual_overseer + }); +} From 515a784abefae6ff79349d0824f4dce84b54ab19 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 17 Sep 2024 12:03:03 +0300 Subject: [PATCH 47/55] Update PRdoc and comments --- .../src/validator_side/collation.rs | 22 +++++++++---------- prdoc/pr_4880.prdoc | 4 ++-- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index a31b640668e5..305e112f0e3f 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -18,14 +18,18 @@ //! //! Usually a path of collations is as follows: //! 1. First, collation must be advertised by collator. +//! 2. The validator inspects the claim queue and decides if the collation should be fetched +//! based on the entries there. A parachain can't have more fetched collations than the +//! entries in the claim queue at a specific relay parent. When calculating this limit the +//! validator counts all advertisements within its view not just at the relay parent. //! 2. If the advertisement was accepted, it's queued for fetch (per relay parent). //! 3. Once it's requested, the collation is said to be Pending. //! 4. Pending collation becomes Fetched once received, we send it to backing for validation. //! 5. If it turns to be invalid or async backing allows seconding another candidate, carry on //! with the next advertisement, otherwise we're done with this relay parent. //! -//! ┌──────────────────────────────────────────┐ -//! └─▶Advertised ─▶ Pending ─▶ Fetched ─▶ Validated +//! ┌───────────────────────────────────┐ +//! └─▶Waiting ─▶ Fetching ─▶ WaitingOnValidation use std::{ collections::{BTreeMap, VecDeque}, @@ -189,9 +193,9 @@ pub struct PendingCollationFetch { pub enum CollationStatus { /// We are waiting for a collation to be advertised to us. Waiting, - /// We are currently fetching a collation. + /// We are currently fetching a collation for the specified `ParaId`. Fetching(ParaId), - /// We are waiting that a collation is being validated. + /// We are waiting that a collation is being validated for the specified `ParaId`. WaitingOnValidation(ParaId), } @@ -207,14 +211,8 @@ pub struct Collations { pub status: CollationStatus, /// Collator we're fetching from, optionally which candidate was requested. /// - /// This is the last fetch for the relay parent. The value is used in - /// `get_next_collation_to_fetch` (called from `dequeue_next_collation_and_fetch`) to determine - /// if the last fetched collation is the same as the one which just finished. If yes - another - /// collation should be fetched. If not - another fetch was already initiated and - /// `get_next_collation_to_fetch` will do nothing. - /// - /// For the reasons above this value is not set to `None` when the fetch is done! Don't use it - /// to check if there is a pending fetch. + /// This is the currently last started fetch, which did not exceed `MAX_UNSHARED_DOWNLOAD_TIME` + /// yet. pub fetching_from: Option<(CollatorId, Option)>, /// Collation that were advertised to us, but we did not yet fetch. Grouped by `ParaId`. waiting_queue: BTreeMap>, diff --git a/prdoc/pr_4880.prdoc b/prdoc/pr_4880.prdoc index 3b16081f5dd1..4c05dcebb179 100644 --- a/prdoc/pr_4880.prdoc +++ b/prdoc/pr_4880.prdoc @@ -4,8 +4,8 @@ doc: - audience: "Node Dev" description: | Implements collation fetching fairness in the validator side of the collator protocol. With - core time if two (or more) parachains share a single core no fairness is guaranteed between - them in terms of collation fetching. The current implementation was accepting up to + core time in place if two (or more) parachains share a single core no fairness is guaranteed + between them in terms of collation fetching. The current implementation was accepting up to `max_candidate_depth + 1` seconded collations per relay parent and once this limit is reached no new collations are accepted. A misbehaving collator can abuse this fact and prevent other collators/parachains from advertising collations by advertising `max_candidate_depth + 1` From 4ef691962e2d788fb2347e239e5613a3e2010afa Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 17 Sep 2024 13:22:04 +0300 Subject: [PATCH 48/55] Combine `seconded_per_para` and `claims_per_para` from collations in a single struct --- .../src/validator_side/collation.rs | 41 ++++++++++++------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 305e112f0e3f..69d263d59d1d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -205,6 +205,15 @@ impl Default for CollationStatus { } } +/// The number of claims in the claim queue and seconded candidates count for a specific `ParaId`. +#[derive(Default, Debug)] +struct CandidatesStatePerPara { + /// How many collations have been seconded. + pub seconded_per_para: usize, + // Claims in the claim queue for the `ParaId`. + pub claims_per_para: usize, +} + /// Information about collations per relay parent. pub struct Collations { /// What is the current status in regards to a collation for this relay parent? @@ -216,12 +225,8 @@ pub struct Collations { pub fetching_from: Option<(CollatorId, Option)>, /// Collation that were advertised to us, but we did not yet fetch. Grouped by `ParaId`. waiting_queue: BTreeMap>, - /// How many collations have been seconded per `ParaId`. - seconded_per_para: BTreeMap, - // Claims per `ParaId` for the assigned core at the relay parent. This information is obtained - // from `GroupAssignments` which contains either the claim queue (if runtime supports it) for - // the core or the `ParaId` of the parachain assigned to the core. - claims_per_para: BTreeMap, + /// Number of seconded candidates and claims in the claim queue per `ParaId`. + candidates_state: BTreeMap, // Represents the claim queue at the relay parent. The `bool` field indicates if a candidate // was seconded for the `ParaId` at the position in question. In other words - if the claim is // 'satisfied'. If the claim queue is not available `claim_queue_state` will be `None`. @@ -240,11 +245,11 @@ impl Collations { /// Once claim queue runtime api is released everywhere this logic won't be needed anymore and /// can be cleaned up. pub(super) fn new(group_assignments: &Vec, has_claim_queue: bool) -> Self { - let mut claims_per_para = BTreeMap::new(); + let mut candidates_state = BTreeMap::::new(); let mut claim_queue_state = Vec::with_capacity(group_assignments.len()); for para_id in group_assignments { - *claims_per_para.entry(*para_id).or_default() += 1; + candidates_state.entry(*para_id).or_default().claims_per_para += 1; claim_queue_state.push((false, *para_id)); } @@ -257,17 +262,16 @@ impl Collations { status: Default::default(), fetching_from: None, waiting_queue: Default::default(), - seconded_per_para: Default::default(), - claims_per_para, + candidates_state, claim_queue_state, } } /// Note a seconded collation for a given para. pub(super) fn note_seconded(&mut self, para_id: ParaId) { - *self.seconded_per_para.entry(para_id).or_default() += 1; + self.candidates_state.entry(para_id).or_default().seconded_per_para += 1; - gum::trace!(target: LOG_TARGET, ?para_id, new_count=*self.seconded_per_para.entry(para_id).or_default(), "Note seconded."); + gum::trace!(target: LOG_TARGET, ?para_id, new_count=self.candidates_state.entry(para_id).or_default().seconded_per_para, "Note seconded."); // and the claim queue state if let Some(claim_queue_state) = self.claim_queue_state.as_mut() { @@ -315,7 +319,7 @@ impl Collations { gum::trace!( target: LOG_TARGET, waiting_queue=?self.waiting_queue, - claims_per_para=?self.claims_per_para, + candidates_state=?self.candidates_state, "Pick a collation to fetch." ); @@ -351,7 +355,11 @@ impl Collations { // Returns the number of seconded collations for the specified `ParaId`. pub(super) fn seconded_and_pending_for_para(&self, para_id: &ParaId) -> usize { - let seconded_for_para = *self.seconded_per_para.get(¶_id).unwrap_or(&0); + let seconded_for_para = self + .candidates_state + .get(¶_id) + .map(|state| state.seconded_per_para) + .unwrap_or_default(); let pending_for_para = self.pending_for_para(para_id); gum::trace!( @@ -367,7 +375,10 @@ impl Collations { // Returns the number of claims in the claim queue for the specified `ParaId`. pub(super) fn claims_for_para(&self, para_id: &ParaId) -> usize { - self.claims_per_para.get(para_id).copied().unwrap_or_default() + self.candidates_state + .get(para_id) + .map(|state| state.claims_per_para) + .unwrap_or_default() } } From bd7174f031c1562f6b428cd7763f7d54dd40088e Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 17 Sep 2024 14:15:33 +0300 Subject: [PATCH 49/55] No need to handle missing claim queue anymore --- .../src/validator_side/collation.rs | 39 +-------------- .../src/validator_side/mod.rs | 49 +++++++++---------- 2 files changed, 23 insertions(+), 65 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 69d263d59d1d..38a2d21a2a5b 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -227,65 +227,28 @@ pub struct Collations { waiting_queue: BTreeMap>, /// Number of seconded candidates and claims in the claim queue per `ParaId`. candidates_state: BTreeMap, - // Represents the claim queue at the relay parent. The `bool` field indicates if a candidate - // was seconded for the `ParaId` at the position in question. In other words - if the claim is - // 'satisfied'. If the claim queue is not available `claim_queue_state` will be `None`. - claim_queue_state: Option>, } impl Collations { - /// `Collations` should work with and without claim queue support. If the claim queue runtime - /// api is available `GroupAssignments` the claim queue. If not - group assignments will contain - /// just one item (what's scheduled on the core). - /// - /// Some of the logic in `Collations` relies on the claim queue and if it is not available - /// fallbacks to another logic. For this reason `Collations` needs to know if claim queue is - /// available or not. - /// - /// Once claim queue runtime api is released everywhere this logic won't be needed anymore and - /// can be cleaned up. - pub(super) fn new(group_assignments: &Vec, has_claim_queue: bool) -> Self { + pub(super) fn new(group_assignments: &Vec) -> Self { let mut candidates_state = BTreeMap::::new(); - let mut claim_queue_state = Vec::with_capacity(group_assignments.len()); for para_id in group_assignments { candidates_state.entry(*para_id).or_default().claims_per_para += 1; - claim_queue_state.push((false, *para_id)); } - // Not optimal but if the claim queue is not available `group_assignments` will have just - // one element. Can be fixed once claim queue api is released everywhere and the fallback - // code is cleaned up. - let claim_queue_state = if has_claim_queue { Some(claim_queue_state) } else { None }; - Self { status: Default::default(), fetching_from: None, waiting_queue: Default::default(), candidates_state, - claim_queue_state, } } /// Note a seconded collation for a given para. pub(super) fn note_seconded(&mut self, para_id: ParaId) { self.candidates_state.entry(para_id).or_default().seconded_per_para += 1; - gum::trace!(target: LOG_TARGET, ?para_id, new_count=self.candidates_state.entry(para_id).or_default().seconded_per_para, "Note seconded."); - - // and the claim queue state - if let Some(claim_queue_state) = self.claim_queue_state.as_mut() { - for (satisfied, assignment) in claim_queue_state { - if *satisfied { - continue - } - - if assignment == ¶_id { - *satisfied = true; - break - } - } - } } /// Adds a new collation to the waiting queue for the relay parent. This function doesn't diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 17ad576b7ef6..8d2f5548394d 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -342,8 +342,8 @@ struct PerRelayParent { } impl PerRelayParent { - fn new(assignments: GroupAssignments, has_claim_queue: bool) -> Self { - let collations = Collations::new(&assignments.current, has_claim_queue); + fn new(assignments: GroupAssignments) -> Self { + let collations = Collations::new(&assignments.current); Self { assignment: assignments, collations } } } @@ -428,15 +428,13 @@ fn is_relay_parent_in_implicit_view( }) } -// Returns the group assignments for the validator and bool indicating if they are obtained from the -// claim queue or not. The latter is used to handle the fall back case when the claim queue api is -// not available in the runtime. +// Returns the group assignments for the validator based on the information in the claim queue async fn assign_incoming( sender: &mut Sender, current_assignments: &mut HashMap, keystore: &KeystorePtr, relay_parent: Hash, -) -> Result<(GroupAssignments, bool)> +) -> Result where Sender: CollatorProtocolSenderTrait, { @@ -463,25 +461,27 @@ where rotation_info.core_for_group(group, cores.len()) } else { gum::trace!(target: LOG_TARGET, ?relay_parent, "Not a validator"); - return Ok((GroupAssignments { current: Vec::new() }, false)) + return Ok(GroupAssignments { current: Vec::new() }) }; - let (paras_now, has_claim_queue) = match fetch_claim_queue(sender, relay_parent) - .await - .map_err(Error::Runtime)? - { + let paras_now = match fetch_claim_queue(sender, relay_parent).await.map_err(Error::Runtime)? { // Runtime supports claim queue - use it - Some(mut claim_queue) => (claim_queue.0.remove(&core_now), true), - // Claim queue is not supported by the runtime - use availability cores instead. - None => ( + Some(mut claim_queue) => claim_queue.0.remove(&core_now), + // Should never happen since claim queue is released everywhere. + None => { + gum::warn!( + target: LOG_TARGET, + ?relay_parent, + "Claim queue is not available, falling back to availability cores", + ); + cores.get(core_now.0 as usize).and_then(|c| match c { CoreState::Occupied(core) => core.next_up_on_available.as_ref().map(|c| [c.para_id].into_iter().collect()), CoreState::Scheduled(core) => Some([core.para_id].into_iter().collect()), CoreState::Free => None, - }), - false, - ), + }) + }, }; let paras_now = paras_now.unwrap_or_else(|| VecDeque::new()); @@ -498,10 +498,7 @@ where } } - Ok(( - GroupAssignments { current: paras_now.into_iter().collect::>() }, - has_claim_queue, - )) + Ok(GroupAssignments { current: paras_now.into_iter().collect::>() }) } fn remove_outgoing( @@ -1211,13 +1208,11 @@ where state.span_per_relay_parent.insert(*leaf, per_leaf_span); } - let (assignments, has_claim_queue_support) = + let assignments = assign_incoming(sender, &mut state.current_assignments, keystore, *leaf).await?; state.active_leaves.insert(*leaf, async_backing_params); - state - .per_relay_parent - .insert(*leaf, PerRelayParent::new(assignments, has_claim_queue_support)); + state.per_relay_parent.insert(*leaf, PerRelayParent::new(assignments)); state .implicit_view @@ -1232,11 +1227,11 @@ where .unwrap_or_default(); for block_hash in allowed_ancestry { if let Entry::Vacant(entry) = state.per_relay_parent.entry(*block_hash) { - let (assignments, has_claim_queue_support) = + let assignments = assign_incoming(sender, &mut state.current_assignments, keystore, *block_hash) .await?; - entry.insert(PerRelayParent::new(assignments, has_claim_queue_support)); + entry.insert(PerRelayParent::new(assignments)); } } } From df6165e6aa090a26693303d303e65c88438d0cda Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 17 Sep 2024 14:55:28 +0300 Subject: [PATCH 50/55] Remove dead code and fix some comments --- .../src/validator_side/collation.rs | 25 +++++++------------ .../src/validator_side/mod.rs | 5 ---- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 38a2d21a2a5b..5c7335297e0c 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -252,8 +252,7 @@ impl Collations { } /// Adds a new collation to the waiting queue for the relay parent. This function doesn't - /// perform any limits check. The caller (`enqueue_collation`) should assure that the collation - /// limit is respected. + /// perform any limits check. The caller should assure that the collation limit is respected. pub(super) fn add_to_waiting_queue(&mut self, collation: (PendingCollation, CollatorId)) { self.waiting_queue.entry(collation.0.para_id).or_default().push_back(collation); } @@ -268,13 +267,8 @@ impl Collations { /// fetch is the first unsatisfied entry from the claim queue for which there is an /// advertisement. /// - /// If claim queue is not supported then `group_assignment` should contain just one element and - /// the score won't matter. In this case collations will be fetched in the order they were - /// received. - /// - /// Note: `group_assignments` is needed just for the fall back logic. It should be removed once - /// claim queue runtime api is released everywhere since it will be redundant - claim queue will - /// already be available in `self.claim_queue_state`. + /// `claim_queue_state` represents the claim queue and a boolean flag indicating if the claim + /// queue entry is fulfilled or not. pub(super) fn pick_a_collation_to_fetch( &mut self, claim_queue_state: Vec<(bool, ParaId)>, @@ -304,15 +298,14 @@ impl Collations { None } - // Returns the number of pending collations for the specified `ParaId`. This function should - // return either 0 or 1. - fn pending_for_para(&self, para_id: &ParaId) -> usize { + // Returns `true` if there is a pending collation for the specified `ParaId`. + fn pending_for_para(&self, para_id: &ParaId) -> bool { match self.status { - CollationStatus::Fetching(pending_para_id) if pending_para_id == *para_id => 1, + CollationStatus::Fetching(pending_para_id) if pending_para_id == *para_id => true, CollationStatus::WaitingOnValidation(pending_para_id) if pending_para_id == *para_id => - 1, - _ => 0, + true, + _ => false, } } @@ -323,7 +316,7 @@ impl Collations { .get(¶_id) .map(|state| state.seconded_per_para) .unwrap_or_default(); - let pending_for_para = self.pending_for_para(para_id); + let pending_for_para = self.pending_for_para(para_id) as usize; gum::trace!( target: LOG_TARGET, diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 8d2f5548394d..0d0ee9906616 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -409,9 +409,6 @@ struct State { /// Aggregated reputation change reputation: ReputationAggregator, - - /// Last known finalized block number - last_finalized_block_num: u32, } fn is_relay_parent_in_implicit_view( @@ -1258,8 +1255,6 @@ where state.fetched_candidates.retain(|k, _| k.relay_parent != removed); state.span_per_relay_parent.remove(&removed); } - - state.last_finalized_block_num = view.finalized_number; } // Remove blocked seconding requests that left the view. From 4c5c2711a46203daabf0885b64689dcbbfecb75c Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 17 Sep 2024 18:36:59 +0300 Subject: [PATCH 51/55] Remove `is_seconded_limit_reached` and use the code directly due to the weird function params --- .../src/validator_side/mod.rs | 53 ++++++------------- 1 file changed, 17 insertions(+), 36 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 0d0ee9906616..853036738ce2 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -1061,13 +1061,26 @@ where ) .map_err(AdvertisementError::Invalid)?; - if is_seconded_limit_reached( + let claims_for_para = per_relay_parent.collations.claims_for_para(¶_id); + let seconded_and_pending_at_ancestors = seconded_and_pending_for_para_in_view( &state.implicit_view, &state.per_relay_parent, - relay_parent, - per_relay_parent, + &relay_parent, ¶_id, - ) { + ); + + gum::trace!( + target: LOG_TARGET, + ?relay_parent, + ?para_id, + claims_for_para, + seconded_and_pending_at_ancestors, + "Checking if seconded limit is reached" + ); + + // Checks if another collation can be accepted. The number of collations that can be seconded + // per parachain is limited by the entries in claim queue for the `ParaId` in question. + if seconded_and_pending_at_ancestors >= claims_for_para { return Err(AdvertisementError::SecondedLimitReached) } @@ -2020,38 +2033,6 @@ async fn handle_collation_fetch_response( result } -/// Checks if another collation can be accepted. The number of collations that can be seconded -/// per parachain is limited by the entries in claim queue for the `ParaId` in question. Besides the -/// seconded collations at the relay parent of the advertisement any pending or seconded collations -/// at previous relay parents (up to `allowed_ancestry_len` blocks back or last finalized block) are -/// also counted towards the limit. -fn is_seconded_limit_reached( - implicit_view: &ImplicitView, - per_relay_parent: &HashMap, - relay_parent: Hash, - relay_parent_state: &PerRelayParent, - para_id: &ParaId, -) -> bool { - let claims_for_para = relay_parent_state.collations.claims_for_para(para_id); - let seconded_and_pending_at_ancestors = seconded_and_pending_for_para_in_view( - implicit_view, - per_relay_parent, - &relay_parent, - para_id, - ); - - gum::trace!( - target: LOG_TARGET, - ?relay_parent, - ?para_id, - claims_for_para, - seconded_and_pending_at_ancestors, - "Checking if seconded limit is reached" - ); - - seconded_and_pending_at_ancestors >= claims_for_para -} - fn seconded_and_pending_for_para_in_view( implicit_view: &ImplicitView, per_relay_parent: &HashMap, From b0e46272c41bdd63fcbb6c885a085458c24efe78 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 17 Sep 2024 18:50:36 +0300 Subject: [PATCH 52/55] Fix comments --- .../collator-protocol/src/validator_side/mod.rs | 16 +++++++--------- .../tests/prospective_parachains.rs | 2 -- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs index 853036738ce2..adf122a042d4 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/mod.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/mod.rs @@ -2033,6 +2033,8 @@ async fn handle_collation_fetch_response( result } +// Returns how many seconded candidates and pending fetches are there within the view at a specific +// relay parent fn seconded_and_pending_for_para_in_view( implicit_view: &ImplicitView, per_relay_parent: &HashMap, @@ -2044,8 +2046,6 @@ fn seconded_and_pending_for_para_in_view( // If the relay parent is not in the view (unconnected candidate) // `known_allowed_relay_parents_under` will return `None`. In this case we still we just count // the candidate at the specified relay parent. - // TODO: what to do when an unconnected candidate becomes connected and potentially we have - // accepted more candidates than the claim queue allows? implicit_view .known_allowed_relay_parents_under(relay_parent, Some(*para_id)) .or(Some(&[*relay_parent])) // if the relay parent is not in view we still want to count it @@ -2060,6 +2060,8 @@ fn seconded_and_pending_for_para_in_view( .unwrap_or(0) } +// Returns the claim queue with a boolean attached to each entry indicating if the position in the +// claim queue has got a corresponding pending fetch or seconded candidate. fn claim_queue_state( relay_parent: &Hash, per_relay_parent: &HashMap, @@ -2096,12 +2098,8 @@ fn claim_queue_state( Some(claim_queue_state) } -/// Returns the next collation to fetch from the `waiting_queue`. -/// -/// This will reset the status back to `Waiting` using [`CollationStatus::back_to_waiting`]. -/// -/// Returns `Some(_)` if there is any collation to fetch, the `status` is not `Seconded` and -/// the passed in `finished_one` is the currently `waiting_collation`. +/// Returns the next collation to fetch from the `waiting_queue` and reset the status back to +/// `Waiting`. fn get_next_collation_to_fetch( finished_one: &(CollatorId, Option), relay_parent: Hash, @@ -2109,7 +2107,7 @@ fn get_next_collation_to_fetch( ) -> Option<(PendingCollation, CollatorId)> { let claim_queue_state = claim_queue_state(&relay_parent, &state.per_relay_parent, &state.implicit_view)?; - let rp_state = state.per_relay_parent.get_mut(&relay_parent)?; // TODO: this is looked up twice + let rp_state = state.per_relay_parent.get_mut(&relay_parent)?; // If finished one does not match waiting_collation, then we already dequeued another fetch // to replace it. diff --git a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs index 8bce4e959915..adab37027cd5 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/tests/prospective_parachains.rs @@ -1444,7 +1444,6 @@ fn collations_outside_limits_are_not_fetched() { test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, keystore } = test_harness; - // Grandparent of head `a`. let head_b = Hash::from_low_u64_be(128); let head_b_num: u32 = 2; @@ -1548,7 +1547,6 @@ fn fair_collation_fetches() { test_harness(ReputationAggregator::new(|_| true), |test_harness| async move { let TestHarness { mut virtual_overseer, keystore } = test_harness; - // Grandparent of head `a`. let head_b = Hash::from_low_u64_be(128); let head_b_num: u32 = 2; From d1cf41d5838e694856cdf42a28b53eecfe13d40c Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Tue, 17 Sep 2024 18:59:25 +0300 Subject: [PATCH 53/55] `pending_for_para` -> `is_pending_for_para` --- .../network/collator-protocol/src/validator_side/collation.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs index 5c7335297e0c..8d2fcb35afec 100644 --- a/polkadot/node/network/collator-protocol/src/validator_side/collation.rs +++ b/polkadot/node/network/collator-protocol/src/validator_side/collation.rs @@ -299,7 +299,7 @@ impl Collations { } // Returns `true` if there is a pending collation for the specified `ParaId`. - fn pending_for_para(&self, para_id: &ParaId) -> bool { + fn is_pending_for_para(&self, para_id: &ParaId) -> bool { match self.status { CollationStatus::Fetching(pending_para_id) if pending_para_id == *para_id => true, CollationStatus::WaitingOnValidation(pending_para_id) @@ -316,7 +316,7 @@ impl Collations { .get(¶_id) .map(|state| state.seconded_per_para) .unwrap_or_default(); - let pending_for_para = self.pending_for_para(para_id) as usize; + let pending_for_para = self.is_pending_for_para(para_id) as usize; gum::trace!( target: LOG_TARGET, From df3a2154812973144955d542960fb508bde46e8c Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Thu, 19 Sep 2024 17:33:44 +0300 Subject: [PATCH 54/55] Fix `0011-async-backing-6-seconds-rate.toml` - set `lookahead` to 3 otherwise the chain can't progress in time --- .../functional/0011-async-backing-6-seconds-rate.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml b/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml index b776622fdce3..e236d6e023c7 100644 --- a/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml +++ b/polkadot/zombienet_tests/functional/0011-async-backing-6-seconds-rate.toml @@ -14,7 +14,7 @@ chain = "rococo-local" allowed_ancestry_len = 2 [relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] - lookahead = 2 + lookahead = 3 group_rotation_frequency = 4 From b70807b278cfe1df8f7731fac30e8677b04a1cb2 Mon Sep 17 00:00:00 2001 From: Tsvetomir Dimitrov Date: Fri, 20 Sep 2024 10:26:02 +0300 Subject: [PATCH 55/55] Set `lookahead` in polkadot/zombienet_tests/elastic_scaling/0002-elastic-scaling-doesnt-break-parachains.toml --- .../0002-elastic-scaling-doesnt-break-parachains.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/polkadot/zombienet_tests/elastic_scaling/0002-elastic-scaling-doesnt-break-parachains.toml b/polkadot/zombienet_tests/elastic_scaling/0002-elastic-scaling-doesnt-break-parachains.toml index 9b3576eaa3c2..2705f69b8c0d 100644 --- a/polkadot/zombienet_tests/elastic_scaling/0002-elastic-scaling-doesnt-break-parachains.toml +++ b/polkadot/zombienet_tests/elastic_scaling/0002-elastic-scaling-doesnt-break-parachains.toml @@ -8,6 +8,7 @@ bootnode = true [relaychain.genesis.runtimeGenesis.patch.configuration.config.scheduler_params] max_validators_per_core = 2 num_cores = 2 + lookahead = 3 [relaychain] default_image = "{{ZOMBIENET_INTEGRATION_TEST_IMAGE}}"