[Feature] Part 1: add TargetList for validator ranking

bin/node/runtime/src/lib.rs

@@ -564,7 +564,8 @@ impl pallet_staking::Config for Runtime {
 	type OffendingValidatorsThreshold = OffendingValidatorsThreshold;
 	type ElectionProvider = ElectionProviderMultiPhase;
 	type GenesisElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
-	type VoterList = BagsList;
+	type VoterList = VoterBagsList;
+	type TargetList = TargetBagsList;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = NominationPools;
 	type WeightInfo = pallet_staking::weights::SubstrateWeight<Runtime>;
@@ -718,14 +719,30 @@ impl pallet_election_provider_multi_phase::Config for Runtime {
 
 parameter_types! {
 	pub const BagThresholds: &'static [u64] = &voter_bags::THRESHOLDS;
+	// TODO: revisit to see if we can generate separate thresholds here
+	pub const BagThresholdsBalance: &'static [u64] = &voter_bags::THRESHOLDS;
 }
 
-impl pallet_bags_list::Config for Runtime {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Runtime {
 	type Event = Event;
+	/// The voter bags-list is loosely kept up to date, and the real source of truth for the score
+	/// of each node is the staking pallet.
 	type ScoreProvider = Staking;
-	type WeightInfo = pallet_bags_list::weights::SubstrateWeight<Runtime>;
 	type BagThresholds = BagThresholds;
 	type Score = VoteWeight;
+	type WeightInfo = pallet_bags_list::weights::SubstrateWeight<Runtime>;
+}
+
+type TargetBagsListInstance = pallet_bags_list::Instance2;
+impl pallet_bags_list::Config<TargetBagsListInstance> for Runtime {
+	type Event = Event;
+	// The bags-list itself will be the source of truth about the approval stakes. This implies that
+	// staking should keep the approval stakes up to date at all times.
+	type ScoreProvider = TargetBagsList;
+	type BagThresholds = BagThresholdsBalance;
+	type Score = Balance;
+	type WeightInfo = pallet_bags_list::weights::SubstrateWeight<Runtime>;
 }
 
 parameter_types! {
@@ -1629,7 +1646,8 @@ construct_runtime!(
 		Gilt: pallet_gilt,
 		Uniques: pallet_uniques,
 		TransactionStorage: pallet_transaction_storage,
-		BagsList: pallet_bags_list,
+		VoterBagsList: pallet_bags_list::<Instance1>,
+		TargetBagsList: pallet_bags_list::<Instance2>,
 		StateTrieMigration: pallet_state_trie_migration,
 		ChildBounties: pallet_child_bounties,
 		Referenda: pallet_referenda,

frame/babe/src/mock.rs

@@ -202,6 +202,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

frame/grandpa/src/mock.rs

@@ -206,6 +206,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

frame/nomination-pools/benchmarking/src/mock.rs

@@ -112,6 +112,7 @@ impl pallet_staking::Config for Runtime {
 		frame_election_provider_support::NoElection<(AccountId, BlockNumber, Staking)>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_bags_list::Pallet<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = Pools;
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

frame/offences/benchmarking/src/mock.rs

@@ -178,6 +178,7 @@ impl pallet_staking::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;

frame/session/benchmarking/src/mock.rs

@@ -175,6 +175,7 @@ impl pallet_staking::Config for Test {
 	type GenesisElectionProvider = Self::ElectionProvider;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type VoterList = pallet_staking::UseNominatorsAndValidatorsMap<Self>;
+	type TargetList = pallet_staking::UseValidatorsMap<Self>;
 	type OnStakerSlash = ();
 	type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig;
 	type WeightInfo = ();

frame/staking/src/lib.rs

@@ -685,6 +685,16 @@ pub struct Nominations<T: Config> {
 	pub suppressed: bool,
 }
 
+/// An unbounded version of `Nominations`, use for some really wacky hacks.
+#[derive(PartialEqNoBound, EqNoBound, Clone, Encode, Decode, RuntimeDebugNoBound, TypeInfo)]
+#[codec(mel_bound())]
+#[scale_info(skip_type_params(T))]
+struct UnboundedNominations<T: Config> {
+	pub targets: Vec<T::AccountId>,
+	pub submitted_in: EraIndex,
+	pub suppressed: bool,
+}
+
 /// The amount of exposure (to slashing) than an individual nominator has.
 #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Encode, Decode, RuntimeDebug, TypeInfo)]
 pub struct IndividualExposure<AccountId, Balance: HasCompact> {

frame/staking/src/mock.rs

@@ -100,7 +100,8 @@ frame_support::construct_runtime!(
 		Staking: pallet_staking::{Pallet, Call, Config<T>, Storage, Event<T>},
 		Session: pallet_session::{Pallet, Call, Storage, Event, Config<T>},
 		Historical: pallet_session::historical::{Pallet, Storage},
-		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>},
+		VoterBagsList: pallet_bags_list::<Instance1>::{Pallet, Call, Storage, Event<T>},
+		TargetBagsList: pallet_bags_list::<Instance2>::{Pallet, Call, Storage, Event<T>},
 	}
 );
 
@@ -233,22 +234,37 @@ impl OnUnbalanced<NegativeImbalanceOf<Test>> for RewardRemainderMock {
 
 const THRESHOLDS: [sp_npos_elections::VoteWeight; 9] =
 	[10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000];
+const THRESHOLDS_BALANCE: [Balance; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000];
 
 parameter_types! {
 	pub static BagThresholds: &'static [sp_npos_elections::VoteWeight] = &THRESHOLDS;
+	// TODO: generate separate thresholds for this
+	pub static BagThresholdsBalance: &'static [sp_npos_elections::Balance] = &THRESHOLDS_BALANCE;
 	pub static MaxNominations: u32 = 16;
 	pub static RewardOnUnbalanceWasCalled: bool = false;
 	pub static LedgerSlashPerEra: (BalanceOf<Test>, BTreeMap<EraIndex, BalanceOf<Test>>) = (Zero::zero(), BTreeMap::new());
 }
 
-impl pallet_bags_list::Config for Test {
+type VoterBagsListInstance = pallet_bags_list::Instance1;
+impl pallet_bags_list::Config<VoterBagsListInstance> for Test {
 	type Event = Event;
 	type WeightInfo = ();
+	// Staking is the source of truth for voter bags list, since they are not kept up to date.
 	type ScoreProvider = Staking;
 	type BagThresholds = BagThresholds;
 	type Score = VoteWeight;
 }
 
+type TargetBagsListInstance = pallet_bags_list::Instance2;
+impl pallet_bags_list::Config<TargetBagsListInstance> for Test {
+	type Event = Event;
+	type WeightInfo = ();
+	// Target bags-list are always kept up to date, and in fact Staking does not know them at all!
+	type ScoreProvider = pallet_bags_list::Pallet<Self, TargetBagsListInstance>;
+	type BagThresholds = BagThresholdsBalance;
+	type Score = Balance;
+}
+
 pub struct OnChainSeqPhragmen;
 impl onchain::Config for OnChainSeqPhragmen {
 	type System = Test;
@@ -257,6 +273,61 @@ impl onchain::Config for OnChainSeqPhragmen {
 	type WeightInfo = ();
 }
 
+pub struct TargetBagsListCompat;
+impl SortedListProvider<AccountId> for TargetBagsListCompat {
+	type Error = <TargetBagsList as SortedListProvider<AccountId>>::Error;
+	type Score = <TargetBagsList as SortedListProvider<AccountId>>::Score;
+
+	fn iter() -> Box<dyn Iterator<Item = AccountId>> {
+		let mut all = TargetBagsList::iter()
+			.map(|x| (x, TargetBagsList::get_score(&x).unwrap_or_default()))
+			.collect::<Vec<_>>();
+		all.sort_by(|a, b| match a.1.partial_cmp(&b.1).unwrap() {
+			std::cmp::Ordering::Equal => b.0.partial_cmp(&a.0).unwrap(),
+			// Question: why rerverse?
+			x @ _ => x.reverse(),
+		});
+		Box::new(all.into_iter().map(|(x, _)| x))
+	}
+	fn iter_from(start: &AccountId) -> Result<Box<dyn Iterator<Item = AccountId>>, Self::Error> {
+		TargetBagsList::iter_from(start)
+	}
+	fn count() -> u32 {
+		TargetBagsList::count()
+	}
+	fn contains(id: &AccountId) -> bool {
+		TargetBagsList::contains(id)
+	}
+	fn on_insert(id: AccountId, weight: Self::Score) -> Result<(), Self::Error> {
+		TargetBagsList::on_insert(id, weight)
+	}
+	fn on_update(id: &AccountId, weight: Self::Score) -> Result<(), Self::Error> {
+		TargetBagsList::on_update(id, weight)
+	}
+	fn get_score(id: &AccountId) -> Result<Self::Score, Self::Error> {
+		TargetBagsList::get_score(id)
+	}
+	fn on_remove(id: &AccountId) -> Result<(), Self::Error> {
+		TargetBagsList::on_remove(id)
+	}
+	fn unsafe_regenerate(
+		all: impl IntoIterator<Item = AccountId>,
+		weight_of: Box<dyn Fn(&AccountId) -> Self::Score>,
+	) -> u32 {
+		TargetBagsList::unsafe_regenerate(all, weight_of)
+	}
+	fn unsafe_clear() {
+		TargetBagsList::unsafe_clear();
+	}
+	fn sanity_check() -> Result<(), &'static str> {
+		TargetBagsList::sanity_check()
+	}
+	#[cfg(feature = "runtime-benchmarks")]
+	fn score_update_worst_case(_who: &AccountId, _is_increase: bool) -> Self::Score {
+		Balance::MAX
+	}
+}
+
 pub struct MockReward {}
 impl OnUnbalanced<PositiveImbalanceOf<Test>> for MockReward {
 	fn on_unbalanced(_: PositiveImbalanceOf<Test>) {
@@ -297,7 +368,8 @@ impl crate::pallet::pallet::Config for Test {
 	type ElectionProvider = onchain::UnboundedExecution<OnChainSeqPhragmen>;
 	type GenesisElectionProvider = Self::ElectionProvider;
 	// NOTE: consider a macro and use `UseNominatorsAndValidatorsMap<Self>` as well.
-	type VoterList = BagsList;
+	type VoterList = VoterBagsList;
+	type TargetList = TargetBagsListCompat;
 	type MaxUnlockingChunks = ConstU32<32>;
 	type OnStakerSlash = OnStakerSlashMock<Test>;
 	type BenchmarkingConfig = TestBenchmarkingConfig;
@@ -893,3 +965,15 @@ pub(crate) fn staking_events_since_last_call() -> Vec<crate::Event<Test>> {
 pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) {
 	(Balances::free_balance(who), Balances::reserved_balance(who))
 }
+
+pub(crate) fn validator_ids() -> Vec<AccountId> {
+	Validators::<Test>::iter().map(|(v, _)| v).collect::<Vec<_>>()
+}
+
+pub(crate) fn nominator_ids() -> Vec<AccountId> {
+	Nominators::<Test>::iter().map(|(n, _)| n).collect::<Vec<_>>()
+}
+
+pub(crate) fn nominator_targets(who: AccountId) -> Vec<AccountId> {
+	Nominators::<Test>::get(&who).map(|n| n.targets).unwrap().into_inner()
+}

frame/staking/src/pallet/impls.rs

@@ -1309,6 +1309,50 @@ impl<T: Config> ScoreProvider<T::AccountId> for Pallet<T> {
 	}
 }
 
+/// A simple sorted list implementation that does not require any additional pallets. Note, this
+/// does not provided validators in sorted ordered. If you desire nominators in a sorted order take
+/// a look at [`pallet-bags-list].
+pub struct UseValidatorsMap<T>(sp_std::marker::PhantomData<T>);
+impl<T: Config> SortedListProvider<T::AccountId> for UseValidatorsMap<T> {
+	type Error = ();
+
+	/// Returns iterator over voter list, which can have `take` called on it.
+	fn iter() -> Box<dyn Iterator<Item = T::AccountId>> {
+		Box::new(Validators::<T>::iter().map(|(v, _)| v))
+	}
+	fn count() -> u32 {
+		Validators::<T>::count()
+	}
+	fn contains(id: &T::AccountId) -> bool {
+		Validators::<T>::contains_key(id)
+	}
+	fn on_insert(_: T::AccountId, _weight: VoteWeight) -> Result<(), Self::Error> {
+		// nothing to do on insert.
+		Ok(())
+	}
+	fn on_update(_: &T::AccountId, _weight: VoteWeight) {
+		// nothing to do on update.
+	}
+	fn on_remove(_: &T::AccountId) {
+		// nothing to do on remove.
+	}
+	fn unsafe_regenerate(
+		_: impl IntoIterator<Item = T::AccountId>,
+		_: Box<dyn Fn(&T::AccountId) -> VoteWeight>,
+	) -> u32 {
+		// nothing to do upon regenerate.
+		0
+	}
+	fn sanity_check() -> Result<(), &'static str> {
+		Ok(())
+	}
+
+	fn unsafe_clear() {
+		Validators::<T>::remove_all();
+	}
+}
+
+
 /// A simple voter list implementation that does not require any additional pallets. Note, this
 /// does not provided nominators in sorted ordered. If you desire nominators in a sorted order take
 /// a look at [`pallet-bags-list].

frame/staking/src/pallet/mod.rs

@@ -184,8 +184,34 @@ pub mod pallet {
 		///
 		/// The changes to nominators are reported to this. Moreover, each validator's self-vote is
 		/// also reported as one independent vote.
+		///
+		/// To keep the load off the chain as much as possible, changes made to the staked amount
+		/// via rewards and slashes are not reported and thus need to be manually fixed by the
+		/// staker. In case of `bags-list`, this always means using `rebag` and `putInFrontOf`.
+		///
+		/// Invariant: what comes out of this list will always be a nominator.
 		type VoterList: SortedListProvider<Self::AccountId, Score = VoteWeight>;
 
+		/// Something that provides a best-effort sorted list of targets aka electable validators,
+		/// used for NPoS election.
+		///
+		/// The changes to the approval stake of each validator are reported to this. This means any
+		/// change to:
+		/// 1. The stake of any validator or nominator.
+		/// 2. The targets of any nominator
+		/// 3. The role of any staker (e.g. validator -> chilled, nominator -> validator, etc)
+		///
+		/// Unlike `VoterList`, the values in this list are always kept up to date with reward and
+		/// slash as well, and thus represent the accurate approval stake of all account being
+		/// nominated by nominators.
+		///
+		/// Note that while at the time of nomination, all targets are checked to be real
+		/// validators, they can chill at any point, and their approval stakes will still be
+		/// recorded. This implies that what comes out of iterating this list MIGHT NOT BE AN ACTIVE
+		/// VALIDATOR.
+		/// WIP
+		type TargetList: SortedListProvider<Self::AccountId, Score = BalanceOf<Self>> = ();
+
 		/// The maximum number of `unlocking` chunks a [`StakingLedger`] can have. Effectively
 		/// determines how many unique eras a staker may be unbonding in.
 		#[pallet::constant]
@@ -298,6 +324,14 @@ pub mod pallet {
 	pub type Nominators<T: Config> =
 		CountedStorageMap<_, Twox64Concat, T::AccountId, Nominations<T>>;
 
+		/// Get the nominator `who`, regardless of being decodable or not.
+		pub(crate) fn get_any(who: &T::AccountId) -> Option<UnboundedNominations<T>> {
+			frame_support::storage::unhashed::get::<UnboundedNominations<T>>(
+				&Nominators::<T>::hashed_key_for(who),
+			)
+		}
+	}
+
 	/// The maximum nominator count before we stop allowing new validators to join.
 	///
 	/// When this value is not set, no limits are enforced.