Companion for Store voters in unsorted bags to get good stake iteration properties

Cargo.toml

@@ -92,6 +92,8 @@ members = [
 	"parachain/test-parachains/adder",
 	"parachain/test-parachains/adder/collator",
 	"utils/staking-miner",
+	"utils/remote-ext-tests/bags-list",
+	"utils/voter-bags",
 ]
 
 # We want to be able to build the bridge relayer without pulling it (and all of its

runtime/common/Cargo.toml

@@ -39,6 +39,7 @@ pallet-treasury = { git = "https://github.com/paritytech/substrate", branch = "m
 pallet-election-provider-multi-phase = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-beefy-mmr = { git = "https://github.com/paritytech/grandpa-bridge-gadget", branch = "master", default-features = false }
 frame-election-provider-support = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+pallet-bags-list = { git = "https://github.com/paritytech/substrate", branch = "zeke-voter-bags-module", default-features = false }
 
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
 pallet-babe = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
@@ -96,6 +97,7 @@ std = [
 	"runtime-parachains/std",
 	"xcm/std",
 	"sp-npos-elections/std",
+	"pallet-bags-list/std"
 ]
 runtime-benchmarks = [
 	"libsecp256k1/hmac",

runtime/common/src/elections.rs

@@ -17,11 +17,13 @@
 //! Code for elections.
 
 use super::{BlockExecutionWeight, BlockLength, BlockWeights};
+use frame_election_provider_support::{SortedListProvider, VoteWeight};
 use frame_support::{
 	parameter_types,
 	weights::{DispatchClass, Weight},
 };
 use sp_runtime::Perbill;
+use sp_std::{boxed::Box, convert::From, marker::PhantomData};
 
 parameter_types! {
 	/// A limit for off-chain phragmen unsigned solution submission.
@@ -89,3 +91,59 @@ impl frame_support::pallet_prelude::Get<Option<(usize, sp_npos_elections::Extend
 		Some((iters, 0))
 	}
 }
+
+/// Implementation of `frame_election_provider_support::SortedListProvider` that updates the
+/// bags-list but uses [`pallet_staking::Nominators`] for `iter`. This is meant to be a transitionary
+/// implementation for runtimes to "test" out the bags-list by keeping it up to date, but not yet
+/// using it for snapshot generation. In contrast, a  "complete" implementation would use bags-list
+/// for `iter`.
+pub struct UseNominatorsAndUpdateBagsList<T>(PhantomData<T>);
+impl<T: pallet_bags_list::Config + pallet_staking::Config> SortedListProvider<T::AccountId>
+	for UseNominatorsAndUpdateBagsList<T>
+{
+	type Error = pallet_bags_list::Error;
+
+	fn iter() -> Box<dyn Iterator<Item = T::AccountId>> {
+		Box::new(pallet_staking::Nominators::<T>::iter().map(|(n, _)| n))
+	}
+
+	fn count() -> u32 {
+		pallet_bags_list::Pallet::<T>::count()
+	}
+
+	fn contains(id: &T::AccountId) -> bool {
+		pallet_bags_list::Pallet::<T>::contains(id)
+	}
+
+	fn on_insert(id: T::AccountId, weight: VoteWeight) -> Result<(), Self::Error> {
+		pallet_bags_list::Pallet::<T>::on_insert(id, weight)
+	}
+
+	fn on_update(id: &T::AccountId, new_weight: VoteWeight) {
+		pallet_bags_list::Pallet::<T>::on_update(id, new_weight);
+	}
+
+	fn on_remove(id: &T::AccountId) {
+		pallet_bags_list::Pallet::<T>::on_remove(id);
+	}
+
+	fn regenerate(
+		all: impl IntoIterator<Item = T::AccountId>,
+		weight_of: Box<dyn Fn(&T::AccountId) -> VoteWeight>,
+	) -> u32 {
+		pallet_bags_list::Pallet::<T>::regenerate(all, weight_of)
+	}
+
+	fn sanity_check() -> Result<(), &'static str> {
+		pallet_bags_list::Pallet::<T>::sanity_check()
+	}
+
+	fn clear(count: Option<u32>) -> u32 {
+		pallet_bags_list::Pallet::<T>::clear(count)
+	}
+
+	#[cfg(feature = "runtime-benchmarks")]
+	fn weight_update_worst_case(who: &T::AccountId, is_increase: bool) -> VoteWeight {
+		pallet_bags_list::Pallet::<T>::weight_update_worst_case(who, is_increase)
+	}
+}

runtime/kusama/Cargo.toml

@@ -72,6 +72,7 @@ pallet-treasury = { git = "https://github.com/paritytech/substrate", branch = "m
 pallet-utility = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-vesting = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 pallet-xcm = { path = "../../xcm/pallet-xcm", default-features = false }
+pallet-bags-list = { git = "https://github.com/paritytech/substrate", branch = "zeke-voter-bags-module", default-features = false }
 frame-election-provider-support = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 
 frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, optional = true }
@@ -174,6 +175,7 @@ std = [
 	"xcm-executor/std",
 	"xcm-builder/std",
 	"frame-election-provider-support/std",
+	"pallet-bags-list/std",
 ]
 runtime-benchmarks = [
 	"runtime-common/runtime-benchmarks",
@@ -211,6 +213,7 @@ runtime-benchmarks = [
 	"hex-literal",
 	"xcm-builder/runtime-benchmarks",
 	"frame-election-provider-support/runtime-benchmarks",
+	"pallet-bags-list/runtime-benchmarks",
 ]
 try-runtime = [
 	"frame-executive/try-runtime",

runtime/kusama/src/lib.rs

@@ -103,6 +103,9 @@ use constants::{currency::*, fee::*, time::*};
 // Weights used in the runtime.
 mod weights;
 
+// Voter bag threshold definitions.
+mod voter_bags;
+
 #[cfg(test)]
 mod tests;
 
@@ -354,6 +357,11 @@ parameter_types! {
 
 	// miner configs
 	pub OffchainRepeat: BlockNumber = 5;
+
+	/// Whilst `UseNominatorsAndUpdateBagsList` or `UseNominatorsMap` is in use, this can still be a
+	/// very large value. Once the `BagsList` is in full motion, staking might open its door to many
+	/// more nominators, and this value should instead be what is a "safe" number (e.g. 22500).
+	pub const VoterSnapshotPerBlock: u32 = u32::MAX;
 }
 
 sp_npos_elections::generate_solution_type!(
@@ -399,6 +407,18 @@ impl pallet_election_provider_multi_phase::Config for Runtime {
 		pallet_collective::EnsureProportionAtLeast<_2, _3, AccountId, CouncilCollective>,
 	>;
 	type WeightInfo = weights::pallet_election_provider_multi_phase::WeightInfo<Self>;
+	type VoterSnapshotPerBlock = VoterSnapshotPerBlock;
+}
+
+parameter_types! {
+	pub const BagThresholds: &'static [u64] = &voter_bags::THRESHOLDS;
+}
+
+impl pallet_bags_list::Config for Runtime {
+	type Event = Event;
+	type VoteWeightProvider = Staking;
+	type WeightInfo = weights::pallet_bags_list::WeightInfo<Runtime>;
+	type BagThresholds = BagThresholds;
 }
 
 fn era_payout(
@@ -505,6 +525,9 @@ impl pallet_staking::Config for Runtime {
 	type EraPayout = EraPayout;
 	type NextNewSession = Session;
 	type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator;
+	// Use the nominators map to iter voters, but also perform the bags-list migration and keep
+	// it up-to-date.
+	type SortedListProvider = runtime_common::elections::UseNominatorsAndUpdateBagsList<Runtime>;
 	type WeightInfo = weights::pallet_staking::WeightInfo<Runtime>;
 }
 
@@ -1462,6 +1485,9 @@ construct_runtime! {
 		// Gilts pallet.
 		Gilt: pallet_gilt::{Pallet, Call, Storage, Event<T>, Config} = 38,
 
+		// Provides a semi-sorted list of nominators for staking.
+		BagsList: pallet_bags_list::{Pallet, Call, Storage, Event<T>} = 39,
+
 		// Parachains pallets. Start indices at 50 to leave room.
 		ParachainsOrigin: parachains_origin::{Pallet, Origin} = 50,
 		Configuration: parachains_configuration::{Pallet, Call, Storage, Config<T>} = 51,
@@ -1521,6 +1547,7 @@ pub type Executive = frame_executive::Executive<
 		TechnicalCommitteeStoragePrefixMigration,
 		TechnicalMembershipStoragePrefixMigration,
 		MigrateTipsPalletPrefix,
+		StakingBagsListMigrationV8,
 	),
 >;
 /// The payload being signed in the transactions.
@@ -1638,6 +1665,25 @@ impl OnRuntimeUpgrade for MigrateTipsPalletPrefix {
 	}
 }
 
+// Migration to generate pallet staking's `SortedListProvider` from pre-existing nominators.
+pub struct StakingBagsListMigrationV8;
+
+impl OnRuntimeUpgrade for StakingBagsListMigrationV8 {
+	fn on_runtime_upgrade() -> frame_support::weights::Weight {
+		pallet_staking::migrations::v8::migrate::<Runtime>()
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn pre_upgrade() -> Result<(), &'static str> {
+		pallet_staking::migrations::v8::pre_migrate::<Runtime>()
+	}
+
+	#[cfg(feature = "try-runtime")]
+	fn post_upgrade() -> Result<(), &'static str> {
+		pallet_staking::migrations::v8::post_migrate::<Runtime>()
+	}
+}
+
 #[cfg(not(feature = "disable-runtime-api"))]
 sp_api::impl_runtime_apis! {
 	impl sp_api::Core<Block> for Runtime {
@@ -1979,6 +2025,7 @@ sp_api::impl_runtime_apis! {
 			list_benchmark!(list, extra, pallet_treasury, Treasury);
 			list_benchmark!(list, extra, pallet_utility, Utility);
 			list_benchmark!(list, extra, pallet_vesting, Vesting);
+			list_benchmark!(list, extra, pallet_bags_list, BagsList);
 
 			let storage_info = AllPalletsWithSystem::storage_info();
 
@@ -2052,6 +2099,7 @@ sp_api::impl_runtime_apis! {
 			add_benchmark!(params, batches, pallet_treasury, Treasury);
 			add_benchmark!(params, batches, pallet_utility, Utility);
 			add_benchmark!(params, batches, pallet_vesting, Vesting);
+			add_benchmark!(params, batches, pallet_bags_list, BagsList);
 
 			if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) }
 			Ok(batches)