From 12872c79ed2d26b460952da6035fcb3d002609ec Mon Sep 17 00:00:00 2001 From: kianenigma Date: Fri, 12 Nov 2021 15:07:46 +0100 Subject: [PATCH 1/7] first draft --- bin/node/runtime/src/lib.rs | 17 +- .../election-provider-multi-phase/src/lib.rs | 108 ++--- .../election-provider-multi-phase/src/mock.rs | 109 ++++- frame/election-provider-support/src/lib.rs | 18 +- frame/offences/benchmarking/src/lib.rs | 8 +- frame/session/benchmarking/src/lib.rs | 10 +- frame/staking/src/benchmarking.rs | 74 +++- frame/staking/src/lib.rs | 72 ++++ frame/staking/src/mock.rs | 10 +- frame/staking/src/pallet/impls.rs | 388 +++++++++++++----- frame/staking/src/pallet/mod.rs | 62 ++- frame/staking/src/tests.rs | 203 +++++---- frame/staking/src/weights.rs | 74 +++- 13 files changed, 816 insertions(+), 337 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 570abe53ed01f..3ff4bc4438a75 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -494,7 +494,7 @@ pallet_staking_reward_curve::build! { parameter_types! { pub const SessionsPerEra: sp_staking::SessionIndex = 6; pub const BondingDuration: pallet_staking::EraIndex = 24 * 28; - pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; // 1/4 the bonding duration. + pub const SlashDeferDuration: pallet_staking::EraIndex = 24 * 7; pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; pub const OffendingValidatorsThreshold: Perbill = Perbill::from_percent(17); @@ -508,7 +508,6 @@ impl onchain::Config for Runtime { } impl pallet_staking::Config for Runtime { - const MAX_NOMINATIONS: u32 = MAX_NOMINATIONS; type Currency = Balances; type UnixTime = Timestamp; type CurrencyToVote = U128CurrencyToVote; @@ -535,6 +534,8 @@ impl pallet_staking::Config for Runtime { // Alternatively, use pallet_staking::UseNominatorsMap to just use the nominators map. // Note that the aforementioned does not scale to a very large number of nominators. type SortedListProvider = BagsList; + // each nominator is allowed a fix number of nomination targets. + type NominationQuota = pallet_staking::FixedNominationQuota; type WeightInfo = pallet_staking::weights::SubstrateWeight; } @@ -563,10 +564,9 @@ parameter_types! { .max .get(DispatchClass::Normal); - // BagsList allows a practically unbounded count of nominators to participate in NPoS elections. - // To ensure we respect memory limits when using the BagsList this must be set to a number of - // voters we know can fit into a single vec allocation. - pub const VoterSnapshotPerBlock: u32 = 10_000; + // Our allocator can handle up to 32 MiB, we limit everything to 8 at most to be safe. + pub const VoterSnapshotSizePerBlock: u32 = 8 * 1024 * 1024; + pub const TargetSnapshotSize: u32 = 1 * 1024 * 1024; } sp_npos_elections::generate_solution_type!( @@ -647,10 +647,11 @@ impl pallet_election_provider_multi_phase::Config for Runtime { pallet_election_provider_multi_phase::SolutionAccuracyOf, OffchainRandomBalancing, >; - type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; type ForceOrigin = EnsureRootOrHalfCouncil; + type VoterSnapshotSizePerBlock = VoterSnapshotSizePerBlock; + type TargetSnapshotSize = TargetSnapshotSize; + type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; type BenchmarkingConfig = BenchmarkConfig; - type VoterSnapshotPerBlock = VoterSnapshotPerBlock; } parameter_types! { diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 80a13aa99fb70..57d868caeaeb1 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -248,7 +248,6 @@ use sp_npos_elections::{ VoteWeight, }; use sp_runtime::{ - traits::Bounded, transaction_validity::{ InvalidTransaction, TransactionPriority, TransactionSource, TransactionValidity, TransactionValidityError, ValidTransaction, @@ -641,14 +640,20 @@ pub mod pallet { #[pallet::constant] type SignedDepositWeight: Get>; - /// The maximum number of voters to put in the snapshot. At the moment, snapshots are only - /// over a single block, but once multi-block elections are introduced they will take place - /// over multiple blocks. + /// The maximum byte-size of voters to put in the snapshot. /// - /// Also, note the data type: If the voters are represented by a `u32` in `type - /// CompactSolution`, the same `u32` is used here to ensure bounds are respected. + /// At the moment, snapshots are only over a single block, but once multi-block elections + /// are introduced they will take place over multiple blocks. #[pallet::constant] - type VoterSnapshotPerBlock: Get>; + type VoterSnapshotSizePerBlock: Get; + + /// The maximum byte size of targets to put in the snapshot. + /// + /// The target snapshot is assumed to always fit in one block, and happens next to voter + /// snapshot. In essence, in a single block, `TargetSnapshotSize + + /// VoterSnapshotSizePerBlock` bytes of data are present in a snapshot. + #[pallet::constant] + type TargetSnapshotSize: Get; /// Handler for the slashed deposits. type SlashHandler: OnUnbalanced>; @@ -772,7 +777,7 @@ pub mod pallet { Self::on_initialize_open_unsigned(enabled, now); T::WeightInfo::on_initialize_open_unsigned() } - } + }, _ => T::WeightInfo::on_initialize_nothing(), } } @@ -831,6 +836,13 @@ pub mod pallet { >::MAXIMUM_VOTES_PER_VOTER, as NposSolution>::LIMIT as u32, ); + + // ---------------------------- + // maximum size of a snapshot should not exceed half the size of our allocator limit. + assert!( + T::VoterSnapshotSizePerBlock::get().saturating_add(T::TargetSnapshotSize::get()) < + sp_core::MAX_POSSIBLE_ALLOCATION / 2 - 1 + ); } } @@ -1293,19 +1305,17 @@ impl Pallet { /// Extracted for easier weight calculation. fn create_snapshot_external( ) -> Result<(Vec, Vec>, u32), ElectionError> { - let target_limit = >::max_value().saturated_into::(); - // for now we have just a single block snapshot. - let voter_limit = T::VoterSnapshotPerBlock::get().saturated_into::(); + let target_limit = Some(T::TargetSnapshotSize::get() as usize); + let voter_limit = Some(T::VoterSnapshotSizePerBlock::get() as usize); let targets = - T::DataProvider::targets(Some(target_limit)).map_err(ElectionError::DataProvider)?; - let voters = - T::DataProvider::voters(Some(voter_limit)).map_err(ElectionError::DataProvider)?; + T::DataProvider::targets(target_limit).map_err(ElectionError::DataProvider)?; + let voters = T::DataProvider::voters(voter_limit).map_err(ElectionError::DataProvider)?; let desired_targets = T::DataProvider::desired_targets().map_err(ElectionError::DataProvider)?; // Defensive-only. - if targets.len() > target_limit || voters.len() > voter_limit { + if voter_limit.map_or(false, |l| voters.len() > l) { debug_assert!(false, "Snapshot limit has not been respected."); return Err(ElectionError::DataProvider("Snapshot too big for submission.")) } @@ -1710,8 +1720,8 @@ mod tests { use super::*; use crate::{ mock::{ - multi_phase_events, roll_to, AccountId, ExtBuilder, MockWeightInfo, MultiPhase, - Runtime, SignedMaxSubmissions, System, TargetIndex, Targets, + multi_phase_events, roll_to, ExtBuilder, MockWeightInfo, MultiPhase, Runtime, + SignedMaxSubmissions, System, }, Phase, }; @@ -1955,70 +1965,6 @@ mod tests { }) } - #[test] - fn snapshot_too_big_failure_onchain_fallback() { - // the `MockStaking` is designed such that if it has too many targets, it simply fails. - ExtBuilder::default().build_and_execute(|| { - Targets::set((0..(TargetIndex::max_value() as AccountId) + 1).collect::>()); - - // Signed phase failed to open. - roll_to(15); - assert_eq!(MultiPhase::current_phase(), Phase::Off); - - // Unsigned phase failed to open. - roll_to(25); - assert_eq!(MultiPhase::current_phase(), Phase::Off); - - // On-chain backup works though. - roll_to(29); - let supports = MultiPhase::elect().unwrap(); - assert!(supports.len() > 0); - }); - } - - #[test] - fn snapshot_too_big_failure_no_fallback() { - // and if the backup mode is nothing, we go into the emergency mode.. - ExtBuilder::default().onchain_fallback(false).build_and_execute(|| { - crate::mock::Targets::set( - (0..(TargetIndex::max_value() as AccountId) + 1).collect::>(), - ); - - // Signed phase failed to open. - roll_to(15); - assert_eq!(MultiPhase::current_phase(), Phase::Off); - - // Unsigned phase failed to open. - roll_to(25); - assert_eq!(MultiPhase::current_phase(), Phase::Off); - - roll_to(29); - let err = MultiPhase::elect().unwrap_err(); - assert_eq!(err, ElectionError::Fallback("NoFallback.")); - assert_eq!(MultiPhase::current_phase(), Phase::Emergency); - }); - } - - #[test] - fn snapshot_too_big_truncate() { - // but if there are too many voters, we simply truncate them. - ExtBuilder::default().build_and_execute(|| { - // we have 8 voters in total. - assert_eq!(crate::mock::Voters::get().len(), 8); - // but we want to take 2. - crate::mock::VoterSnapshotPerBlock::set(2); - - // Signed phase opens just fine. - roll_to(15); - assert_eq!(MultiPhase::current_phase(), Phase::Signed); - - assert_eq!( - MultiPhase::snapshot_metadata().unwrap(), - SolutionOrSnapshotSize { voters: 2, targets: 4 } - ); - }) - } - #[test] fn untrusted_score_verification_is_respected() { ExtBuilder::default().build_and_execute(|| { diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index fbde6ad991706..c3d13a65b494a 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -268,7 +268,17 @@ parameter_types! { pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxLength: u32 = 256; pub static MockWeightInfo: bool = false; - pub static VoterSnapshotPerBlock: VoterIndex = u32::max_value(); + pub static VoterSnapshotSizePerBlock: u32 = 4 * 1024 * 1024; + pub static TargetSnapshotSize: u32 = 1 * 1024 * 1024; + + // TODO: we want a test to see: + // - how many nominators we can create in a normal distribution, for a runtime + // - what is the maximum number of nominators that we can create, for a runtime. + // - with this work we are potentially allowing more NUMBER of nominators, with the same amount + // of byte size to enter the system. We need to make sure nothing else breaks elsewhere down + // the road. + // - make sure both of the above run in wasm. + // - For all of this, we need a `pallet-election-playground`. pub static EpochLength: u64 = 30; pub static OnChianFallback: bool = true; @@ -402,7 +412,8 @@ impl crate::Config for Runtime { type Fallback = MockFallback; type ForceOrigin = frame_system::EnsureRoot; type Solution = TestNposSolution; - type VoterSnapshotPerBlock = VoterSnapshotPerBlock; + type VoterSnapshotSizePerBlock = VoterSnapshotSizePerBlock; + type TargetSnapshotSize = TargetSnapshotSize; type Solver = SequentialPhragmen, Balancing>; } @@ -422,10 +433,10 @@ pub struct ExtBuilder {} pub struct StakingMock; impl ElectionDataProvider for StakingMock { const MAXIMUM_VOTES_PER_VOTER: u32 = ::LIMIT as u32; - fn targets(maybe_max_len: Option) -> data_provider::Result> { + fn targets(maybe_max_size: Option) -> data_provider::Result> { let targets = Targets::get(); - if maybe_max_len.map_or(false, |max_len| targets.len() > max_len) { + if maybe_max_size.map_or(false, |max_len| targets.encoded_size() > max_len) { return Err("Targets too big") } @@ -433,11 +444,19 @@ impl ElectionDataProvider for StakingMock { } fn voters( - maybe_max_len: Option, + maybe_max_size: Option, ) -> data_provider::Result)>> { let mut voters = Voters::get(); - if let Some(max_len) = maybe_max_len { - voters.truncate(max_len) + + if let Some(max_len) = maybe_max_size { + while voters.encoded_size() > max_len && !voters.is_empty() { + log::trace!( + target: crate::LOG_TARGET, + "staking-mock: truncating length to {}", + voters.len() - 1 + ); + let _ = voters.pop(); + } } Ok(voters) @@ -488,6 +507,82 @@ impl ElectionDataProvider for StakingMock { } } +#[cfg(test)] +mod staking_mock { + use super::*; + + #[test] + fn snapshot_too_big_failure_onchain_fallback() { + // the `StakingMock` is designed such that if it has too many targets, it simply fails. + ExtBuilder::default().build_and_execute(|| { + // 1 byte won't allow for anything. + TargetSnapshotSize::set(1); + + // Signed phase failed to open. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + // Unsigned phase failed to open. + roll_to(25); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + // On-chain backup works though. + roll_to(29); + let supports = MultiPhase::elect().unwrap(); + assert!(supports.len() > 0); + }); + } + + #[test] + fn snapshot_too_big_failure_no_fallback() { + // .. and if the backup mode is nothing, we go into the emergency mode.. + ExtBuilder::default().onchain_fallback(false).build_and_execute(|| { + TargetSnapshotSize::set(1); + + // Signed phase failed to open. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + // Unsigned phase failed to open. + roll_to(25); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + // emergency phase has started. + roll_to(29); + let err = MultiPhase::elect().unwrap_err(); + assert_eq!(err, ElectionError::Fallback("NoFallback.")); + assert_eq!(MultiPhase::current_phase(), Phase::Emergency); + }); + } + + #[test] + fn snapshot_voter_too_big_truncate() { + // if there are too many voters, we simply truncate them. + ExtBuilder::default().build_and_execute(|| { + // we have 8 voters in total. + assert_eq!(crate::mock::Voters::get().len(), 8); + + // the byte-size of taking the first two voters. + let limit_2 = crate::mock::Voters::get() + .into_iter() + .take(2) + .collect::>() + .encoded_size() as u32; + crate::mock::VoterSnapshotSizePerBlock::set(limit_2); + + // Signed phase opens just fine. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Signed); + + // snapshot has only two voters. + assert_eq!( + MultiPhase::snapshot_metadata().unwrap(), + SolutionOrSnapshotSize { voters: 2, targets: 4 } + ); + }) + } +} + impl ExtBuilder { pub fn miner_tx_priority(self, p: u64) -> Self { ::set(p); diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index cb36e025c3bee..387a6edee447e 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -180,28 +180,30 @@ pub mod data_provider { /// Something that can provide the data to an [`ElectionProvider`]. pub trait ElectionDataProvider { /// Maximum number of votes per voter that this data provider is providing. + /// + /// Note that this is the absolute maximum, less votes is also possible. const MAXIMUM_VOTES_PER_VOTER: u32; /// All possible targets for the election, i.e. the candidates. /// - /// If `maybe_max_len` is `Some(v)` then the resulting vector MUST NOT be longer than `v` items - /// long. + /// If `maybe_max_size` is `Some(v)` then the size of the resulting vector MUST NOT be more than + /// `v` bytes. /// /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. - fn targets(maybe_max_len: Option) -> data_provider::Result>; + fn targets(maybe_max_size: Option) -> data_provider::Result>; /// All possible voters for the election. /// /// Note that if a notion of self-vote exists, it should be represented here. /// - /// If `maybe_max_len` is `Some(v)` then the resulting vector MUST NOT be longer than `v` items - /// long. + /// If `maybe_max_size` is `Some(v)` then the size of the resulting vector MUST NOT be more than + /// `v` bytes. /// /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. fn voters( - maybe_max_len: Option, + maybe_max_size: Option, ) -> data_provider::Result)>>; /// The number of targets to elect. @@ -250,11 +252,11 @@ pub trait ElectionDataProvider { #[cfg(feature = "std")] impl ElectionDataProvider for () { const MAXIMUM_VOTES_PER_VOTER: u32 = 0; - fn targets(_maybe_max_len: Option) -> data_provider::Result> { + fn targets(_: Option) -> data_provider::Result> { Ok(Default::default()) } fn voters( - _maybe_max_len: Option, + _: Option, ) -> data_provider::Result)>> { Ok(Default::default()) } diff --git a/frame/offences/benchmarking/src/lib.rs b/frame/offences/benchmarking/src/lib.rs index c920b0b900dff..6e31d47062212 100644 --- a/frame/offences/benchmarking/src/lib.rs +++ b/frame/offences/benchmarking/src/lib.rs @@ -43,7 +43,7 @@ use pallet_session::{ Config as SessionConfig, SessionManager, }; use pallet_staking::{ - Config as StakingConfig, Event as StakingEvent, Exposure, IndividualExposure, + Config as StakingConfig, Event as StakingEvent, Exposure, IndividualExposure, NominationQuota, Pallet as Staking, RewardDestination, ValidatorPrefs, }; @@ -275,7 +275,7 @@ benchmarks! { let r in 1 .. MAX_REPORTERS; // we skip 1 offender, because in such case there is no slashing let o in 2 .. MAX_OFFENDERS; - let n in 0 .. MAX_NOMINATORS.min(::MAX_NOMINATIONS); + let n in 0 .. MAX_NOMINATORS.min(::NominationQuota::ABSOLUTE_MAXIMUM); // Make r reporters let mut reporters = vec![]; @@ -381,7 +381,7 @@ benchmarks! { } report_offence_grandpa { - let n in 0 .. MAX_NOMINATORS.min(::MAX_NOMINATIONS); + let n in 0 .. MAX_NOMINATORS.min(::NominationQuota::ABSOLUTE_MAXIMUM); // for grandpa equivocation reports the number of reporters // and offenders is always 1 @@ -416,7 +416,7 @@ benchmarks! { } report_offence_babe { - let n in 0 .. MAX_NOMINATORS.min(::MAX_NOMINATIONS); + let n in 0 .. MAX_NOMINATORS.min(::NominationQuota::ABSOLUTE_MAXIMUM); // for babe equivocation reports the number of reporters // and offenders is always 1 diff --git a/frame/session/benchmarking/src/lib.rs b/frame/session/benchmarking/src/lib.rs index 8ca713b1bbf61..11cc505888556 100644 --- a/frame/session/benchmarking/src/lib.rs +++ b/frame/session/benchmarking/src/lib.rs @@ -33,7 +33,7 @@ use frame_system::RawOrigin; use pallet_session::{historical::Module as Historical, Pallet as Session, *}; use pallet_staking::{ benchmarking::create_validator_with_nominators, testing_utils::create_validators, - RewardDestination, + NominationQuota, RewardDestination, }; use sp_runtime::traits::{One, StaticLookup}; @@ -53,10 +53,10 @@ impl OnInitialize for Pallet { benchmarks! { set_keys { - let n = ::MAX_NOMINATIONS; + let n = ::NominationQuota::ABSOLUTE_MAXIMUM; let (v_stash, _) = create_validator_with_nominators::( n, - ::MAX_NOMINATIONS, + ::NominationQuota::ABSOLUTE_MAXIMUM, false, RewardDestination::Staked, )?; @@ -69,10 +69,10 @@ benchmarks! { }: _(RawOrigin::Signed(v_controller), keys, proof) purge_keys { - let n = ::MAX_NOMINATIONS; + let n = ::NominationQuota::ABSOLUTE_MAXIMUM; let (v_stash, _) = create_validator_with_nominators::( n, - ::MAX_NOMINATIONS, + ::NominationQuota::ABSOLUTE_MAXIMUM, false, RewardDestination::Staked )?; diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 80630818de7e6..18cdf56bb8a99 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -354,17 +354,17 @@ benchmarks! { kick { // scenario: we want to kick `k` nominators from nominating us (we are a validator). // we'll assume that `k` is under 128 for the purposes of determining the slope. - // each nominator should have `T::MAX_NOMINATIONS` validators nominated, and our validator + // each nominator should have `T::NominationQuota::ABSOLUTE_MAXIMUM` validators nominated, and our validator // should be somewhere in there. let k in 1 .. 128; - // these are the other validators; there are `T::MAX_NOMINATIONS - 1` of them, so - // there are a total of `T::MAX_NOMINATIONS` validators in the system. - let rest_of_validators = create_validators_with_seed::(T::MAX_NOMINATIONS - 1, 100, 415)?; + // these are the other validators; there are `T::NominationQuota::ABSOLUTE_MAXIMUM - 1` of them, so + // there are a total of `T::NominationQuota::ABSOLUTE_MAXIMUM` validators in the system. + let rest_of_validators = create_validators_with_seed::(T::NominationQuota::ABSOLUTE_MAXIMUM - 1, 100, 415)?; // this is the validator that will be kicking. let (stash, controller) = create_stash_controller::( - T::MAX_NOMINATIONS - 1, + T::NominationQuota::ABSOLUTE_MAXIMUM - 1, 100, Default::default(), )?; @@ -379,7 +379,7 @@ benchmarks! { for i in 0 .. k { // create a nominator stash. let (n_stash, n_controller) = create_stash_controller::( - T::MAX_NOMINATIONS + i, + T::NominationQuota::ABSOLUTE_MAXIMUM + i, 100, Default::default(), )?; @@ -414,9 +414,9 @@ benchmarks! { } } - // Worst case scenario, T::MAX_NOMINATIONS + // Worst case scenario, T::NominationQuota::ABSOLUTE_MAXIMUM nominate { - let n in 1 .. T::MAX_NOMINATIONS; + let n in 1 .. T::NominationQuota::ABSOLUTE_MAXIMUM; // clean up any existing state. clear_validators_and_nominators::(); @@ -427,7 +427,7 @@ benchmarks! { // we are just doing an insert into the origin position. let scenario = ListScenario::::new(origin_weight, true)?; let (stash, controller) = create_stash_controller_with_balance::( - SEED + T::MAX_NOMINATIONS + 1, // make sure the account does not conflict with others + SEED + T::NominationQuota::ABSOLUTE_MAXIMUM + 1, // make sure the account does not conflict with others origin_weight, Default::default(), ).unwrap(); @@ -714,7 +714,7 @@ benchmarks! { create_validators_with_nominators_for_era::( v, n, - ::MAX_NOMINATIONS as usize, + ::NominationQuota::ABSOLUTE_MAXIMUM as usize, false, None, )?; @@ -732,7 +732,7 @@ benchmarks! { create_validators_with_nominators_for_era::( v, n, - ::MAX_NOMINATIONS as usize, + ::NominationQuota::ABSOLUTE_MAXIMUM as usize, false, None, )?; @@ -803,7 +803,7 @@ benchmarks! { assert!(balance_before > balance_after); } - get_npos_voters { + get_npos_voters_unbounded { // number of validator intention. let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; // number of nominator intention. @@ -812,7 +812,7 @@ benchmarks! { let s in 1 .. 20; let validators = create_validators_with_nominators_for_era::( - v, n, T::MAX_NOMINATIONS as usize, false, None + v, n, T::NominationQuota::ABSOLUTE_MAXIMUM as usize, false, None )? .into_iter() .map(|v| T::Lookup::lookup(v).unwrap()) @@ -824,21 +824,56 @@ benchmarks! { let num_voters = (v + n) as usize; }: { - let voters = >::get_npos_voters(None); + let voters = >::get_npos_voters_unbounded(); assert_eq!(voters.len(), num_voters); } - get_npos_targets { + get_npos_voters_bounded { // number of validator intention. let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; // number of nominator intention. - let n = MAX_NOMINATORS; + let n in (MAX_NOMINATORS / 2) .. MAX_NOMINATORS; + // total number of slashing spans. Assigned to validators randomly. + let s in 1 .. 20; + + let validators = create_validators_with_nominators_for_era::( + v, n, T::NominationQuota::ABSOLUTE_MAXIMUM as usize, false, None + )? + .into_iter() + .map(|v| T::Lookup::lookup(v).unwrap()) + .collect::>(); + + (0..s).for_each(|index| { + add_slashing_spans::(&validators[index as usize], 10); + }); + + let num_voters = (v + n) as usize; + }: { + let voters = >::get_npos_voters_bounded(Bounded::max_value()); + assert_eq!(voters.len(), num_voters); + } + + get_npos_targets_unbounded { + // number of validator intention. + let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; + + let _ = create_validators_with_nominators_for_era::( + v, 0, 0, false, None + )?; + }: { + let targets = Validators::::iter().map(|(v, _)| v).collect::>(); + assert_eq!(targets.len() as u32, v); + } + + get_npos_targets_bounded { + // number of validator intention. + let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; let _ = create_validators_with_nominators_for_era::( - v, n, T::MAX_NOMINATIONS as usize, false, None + v, 0, 0, false, None )?; }: { - let targets = >::get_npos_targets(); + let targets = >::get_npos_targets_bounded(Bounded::max_value()); assert_eq!(targets.len() as u32, v); } @@ -910,7 +945,8 @@ mod tests { create_validators_with_nominators_for_era::( v, n, - ::MAX_NOMINATIONS as usize, + <::NominationQuota as NominationQuota>>::ABSOLUTE_MAXIMUM + as usize, false, None, ) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index be02e8d91d326..79af80dd23245 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -799,3 +799,75 @@ where R::is_known_offence(offenders, time_slot) } } + +/// Something that can dictate the maximum number of nominations per nominator. +pub trait NominationQuota { + /// The absolute maximum number that this trait can return. + // NOTE: this is useful for generating the compact solution type, i.e. + // `sp_npos_elections::generate_solution_type`. + const ABSOLUTE_MAXIMUM: u32; + + /// Determine the number of nominations that an account with `balance` at stake is allowed to + /// have. + fn nomination_quota(balance: Balance) -> u32; +} + +/// A nomination quota descriptor that allows `MAX` for all nominators. +pub struct FixedNominationQuota; +impl NominationQuota for FixedNominationQuota { + const ABSOLUTE_MAXIMUM: u32 = MAX; + + fn nomination_quota(_: Balance) -> u32 { + MAX + } +} + +use sp_runtime::{ + traits::{One, UniqueSaturatedInto}, + SaturatedConversion, +}; + +/// An implementation of a linear nomination distribution. +/// +/// TODO: ideally we want this to be a const expression all the way through, no point in +/// re-computing this on the fly. For now we stick to a runtime placeholder. +/// +/// Essentially, draws a line between `(MinBalance, MinQuota)` and `(MaxBalance, MaxQuota)`, Also, +/// values less than `MinBalance` have `MinQuota` and values higher than `MaxBalance` have +/// `MaxQuota`. +pub struct LinearNominationQuota( + sp_std::marker::PhantomData<(MinBalance, MaxBalance)>, +); +impl + NominationQuota for LinearNominationQuota +where + MinBalance: Get, + MaxBalance: Get, + Balance: From + + Copy + + UniqueSaturatedInto + + One + + Ord + + sp_std::ops::Sub + + sp_std::ops::Div, +{ + const ABSOLUTE_MAXIMUM: u32 = MAX_QUOTA; + + fn nomination_quota(balance: Balance) -> u32 { + if balance < MinBalance::get() { + MIN_QUOTA + } else if balance >= MinBalance::get() && balance < MaxBalance::get() { + let min_quota: Balance = MIN_QUOTA.into(); + let max_quota: Balance = MAX_QUOTA.into(); + // per this much balance, the quota increases per one (reverse of slope). + let balance_per_quota_inc = + (balance - MinBalance::get()) / (min_quota - max_quota).max(One::one()); + // this many increments happens to the quota. + let inc_steps = (balance - MinBalance::get()) / balance_per_quota_inc.max(One::one()); + + MIN_QUOTA + inc_steps.saturated_into::() + } else { + MAX_QUOTA + } + } +} diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 95d397359f8d6..6e2e90aa22f5b 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -252,7 +252,6 @@ impl onchain::Config for Test { } impl crate::pallet::pallet::Config for Test { - const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; type UnixTime = Timestamp; type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -271,6 +270,7 @@ impl crate::pallet::pallet::Config for Test { type OffendingValidatorsThreshold = OffendingValidatorsThreshold; type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; + type NominationQuota = FixedNominationQuota<16>; type WeightInfo = (); // NOTE: consider a macro and use `UseNominatorsMap` as well. type SortedListProvider = BagsList; @@ -853,3 +853,11 @@ pub(crate) fn staking_events() -> Vec> { pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { (Balances::free_balance(who), Balances::reserved_balance(who)) } + +pub(crate) fn validator_ids() -> Vec { + Validators::::iter().map(|(v, _)| v).collect() +} + +pub(crate) fn nominator_ids() -> Vec { + Nominators::::iter().map(|(n, _)| n).collect() +} diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 7ca1cb1a4a61b..ec639406c210a 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -43,8 +43,8 @@ use sp_std::{collections::btree_map::BTreeMap, prelude::*}; use crate::{ log, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, Exposure, - ExposureOf, Forcing, IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, - SessionInterface, StakingLedger, ValidatorPrefs, + ExposureOf, Forcing, IndividualExposure, NominationQuota, Nominations, PositiveImbalanceOf, + RewardDestination, SessionInterface, StakingLedger, ValidatorPrefs, }; use super::{pallet::*, STAKING_ID}; @@ -647,120 +647,203 @@ impl Pallet { SlashRewardFraction::::put(fraction); } - /// Get all of the voters that are eligible for the npos election. + /// Get all of the voters that are eligible for the next npos election. /// - /// `maybe_max_len` can imposes a cap on the number of voters returned; First all the validator - /// are included in no particular order, then remainder is taken from the nominators, as - /// returned by [`Config::SortedListProvider`]. + /// This function is self-weighing as [`DispatchClass::Mandatory`]. /// - /// This will use nominators, and all the validators will inject a self vote. + /// # Warning + /// + /// This is the unbounded variant. Being called might cause a large number of storage reads. Use + /// [`get_npos_targets_bounded`] otherwise. + pub fn get_npos_voters_unbounded() -> Vec<(T::AccountId, VoteWeight, Vec)> { + let slashing_spans = >::iter().collect::>(); + let weight_of = Self::weight_of_fn(); + + let mut nominator_votes = T::SortedListProvider::iter() + .filter_map(|nominator| { + if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = + Self::nominators(nominator.clone()) + { + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + if !targets.len().is_zero() { + let weight = weight_of(&nominator); + return Some((nominator, weight, targets)) + } + } + None + }) + .collect::>(); + + let validator_votes = >::iter() + .map(|(v, _)| (v.clone(), Self::weight_of(&v), vec![v.clone()])) + .collect::>(); + + Self::register_weight(T::WeightInfo::get_npos_voters_unbounded( + validator_votes.len() as u32, + nominator_votes.len() as u32, + slashing_spans.len() as u32, + )); + log!( + info, + "generated {} npos voters, {} from validators and {} nominators, without size limit", + validator_votes.len() + nominator_votes.len(), + validator_votes.len(), + nominator_votes.len(), + ); + + // NOTE: we chain the one we expect to have the smaller size (`validators_votes`) to the + // larger one, to minimize copying. Ideally we would collect only once, but sadly then we + // wouldn't have access to a cheap `.len()`, which we need for weighing. TODO: maybe + // `.count()` is still cheaper than the copying we do. + nominator_votes.extend(validator_votes); + nominator_votes + } + + /// Get all of the voters that are eligible for the next npos election. + /// + /// `max_size` imposes a cap on the byte-size of the entire voters returned. + /// + /// As of now, first all the validator are included in no particular order, then remainder is + /// taken from the nominators, as returned by [`Config::SortedListProvider`]. /// /// This function is self-weighing as [`DispatchClass::Mandatory`]. /// /// ### Slashing /// /// All nominations that have been submitted before the last non-zero slash of the validator are - /// auto-chilled, but still count towards the limit imposed by `maybe_max_len`. - pub fn get_npos_voters( - maybe_max_len: Option, + /// auto-chilled, and they DO count towards the limit imposed by `maybe_max_size`. + /// + /// In essence, this implementation ensures that no more than `max_size` bytes worth of npos + /// voters are READ from storage, which subsequently ensures that the returning value cannot be + /// more than `max_size`. + pub fn get_npos_voters_bounded( + max_size: usize, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { - let max_allowed_len = { - let nominator_count = CounterForNominators::::get() as usize; - let validator_count = CounterForValidators::::get() as usize; - let all_voter_count = validator_count.saturating_add(nominator_count); - maybe_max_len.unwrap_or(all_voter_count).min(all_voter_count) - }; - - let mut all_voters = Vec::<_>::with_capacity(max_allowed_len); + let mut tracker = StaticSizeTracker::::new(); + let mut voters = Vec::<_>::with_capacity( + max_size / + StaticSizeTracker::::voter_size( + T::NominationQuota::ABSOLUTE_MAXIMUM as usize, + ), + ); - // first, grab all validators in no particular order, capped by the maximum allowed length. - let mut validators_taken = 0u32; - for (validator, _) in >::iter().take(max_allowed_len) { + // first, grab all validators in no particular order. In most cases, all of them should fit + // anyway. + for (validator, _) in >::iter() { // Append self vote. let self_vote = (validator.clone(), Self::weight_of(&validator), vec![validator.clone()]); - all_voters.push(self_vote); - validators_taken.saturating_inc(); + tracker.register_voter(1usize); + if tracker.final_byte_size_of(voters.len() + 1) > max_size { + log!( + warn, + "stopped iterating over validators' self-vote at {} to cap the size at {}. This should probably never happen", + voters.len(), + max_size, + ); + break + } + voters.push(self_vote); } + let validators_taken = voters.len(); - // .. and grab whatever we have left from nominators. - let nominators_quota = (max_allowed_len as u32).saturating_sub(validators_taken); + // cache a few items. let slashing_spans = >::iter().collect::>(); - - // track the count of nominators added to `all_voters - let mut nominators_taken = 0u32; - // track every nominator iterated over, but not necessarily added to `all_voters` - let mut nominators_seen = 0u32; - - // cache the total-issuance once in this function let weight_of = Self::weight_of_fn(); - let mut nominators_iter = T::SortedListProvider::iter(); - while nominators_taken < nominators_quota && nominators_seen < nominators_quota * 2 { - let nominator = match nominators_iter.next() { - Some(nominator) => { - nominators_seen.saturating_inc(); - nominator - }, - None => break, - }; - + for nominator in T::SortedListProvider::iter() { if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = >::get(&nominator) { - log!( - trace, - "fetched nominator {:?} with weight {:?}", - nominator, - weight_of(&nominator) - ); + // IMPORTANT: we track the size and potentially break out right here. + tracker.register_voter(targets.len()); + if tracker.final_byte_size_of(voters.len() + 1) > max_size { + break + } + targets.retain(|stash| { slashing_spans .get(stash) .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) }); if !targets.len().is_zero() { - all_voters.push((nominator.clone(), weight_of(&nominator), targets)); - nominators_taken.saturating_inc(); + voters.push((nominator.clone(), weight_of(&nominator), targets)); } } else { log!(error, "DEFENSIVE: invalid item in `SortedListProvider`: {:?}", nominator) } } - // all_voters should have not re-allocated. - debug_assert!(all_voters.capacity() == max_allowed_len); - - Self::register_weight(T::WeightInfo::get_npos_voters( - validators_taken, - nominators_taken, + let nominators_taken = voters.len().saturating_sub(validators_taken); + Self::register_weight(T::WeightInfo::get_npos_voters_bounded( + validators_taken as u32, + nominators_taken as u32, slashing_spans.len() as u32, )); + debug_assert!(voters.encoded_size() <= max_size); log!( info, - "generated {} npos voters, {} from validators and {} nominators", - all_voters.len(), + "generated {} npos voters, {} from validators and {} nominators, with size limit {}", + voters.len(), validators_taken, - nominators_taken + nominators_taken, + max_size, ); - all_voters + voters } - /// Get the targets for an upcoming npos election. + /// Get the list of targets (validators) that are eligible for the next npos election. + // This function is self-weighing as [`DispatchClass::Mandatory`]. + /// + /// # Warning + /// + /// This is the unbounded variant. Being called might cause a large number of storage reads. Use + /// [`get_npos_targets_bounded`] otherwise. + pub fn get_npos_targets_unbounded() -> Vec { + let targets = Validators::::iter().map(|(v, _)| v).collect::>(); + Self::register_weight(T::WeightInfo::get_npos_targets_unbounded(targets.len() as u32)); + log!(info, "generated {} npos targets, without size limit.", targets.len()); + targets + } + + /// Get the list of targets (validators) that are eligible for the next npos election. + /// + /// `max_size` imposes a cap on the byte-size of the entire targets returned. /// /// This function is self-weighing as [`DispatchClass::Mandatory`]. - pub fn get_npos_targets() -> Vec { - let mut validator_count = 0u32; - let targets = Validators::::iter() - .map(|(v, _)| { - validator_count.saturating_inc(); - v - }) - .collect::>(); + pub fn get_npos_targets_bounded(max_size: usize) -> Vec { + let mut internal_size: usize = Zero::zero(); + let mut targets: Vec = + Vec::with_capacity(max_size / sp_std::mem::size_of::()); + + for (next, _) in Validators::::iter() { + let new_internal_size = internal_size + sp_std::mem::size_of::(); + let new_final_size = new_internal_size + + StaticSizeTracker::::length_prefix(targets.len() + 1); + if new_final_size > max_size { + // we've had enough + break + } + targets.push(next); + internal_size = new_internal_size; + + debug_assert_eq!(targets.encoded_size(), new_final_size); + } - Self::register_weight(T::WeightInfo::get_npos_targets(validator_count)); + Self::register_weight(T::WeightInfo::get_npos_targets_bounded(targets.len() as u32)); + debug_assert!( + targets.encoded_size() <= max_size, + "encoded size: {}, max_size: {}", + targets.encoded_size(), + max_size + ); + log!(info, "generated {} npos targets, with size limit {}", targets.len(), max_size); targets } @@ -855,7 +938,7 @@ impl Pallet { } impl ElectionDataProvider> for Pallet { - const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; + const MAXIMUM_VOTES_PER_VOTER: u32 = T::NominationQuota::ABSOLUTE_MAXIMUM; fn desired_targets() -> data_provider::Result { Self::register_weight(T::DbWeight::get().reads(1)); @@ -863,32 +946,39 @@ impl ElectionDataProvider> for Pallet } fn voters( - maybe_max_len: Option, + maybe_max_size: Option, ) -> data_provider::Result)>> { - debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); - debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); - debug_assert_eq!( - CounterForNominators::::get(), - T::SortedListProvider::count(), - "voter_count must be accurate", - ); - - // This can never fail -- if `maybe_max_len` is `Some(_)` we handle it. - let voters = Self::get_npos_voters(maybe_max_len); - debug_assert!(maybe_max_len.map_or(true, |max| voters.len() <= max)); - - Ok(voters) + Ok(match maybe_max_size { + Some(max_size) => Self::get_npos_voters_bounded(max_size), + None => { + log!( + warn, + "iterating over an unbounded number of npos voters, this might exhaust the \ + memory limits of the chain. Ensure proper limits are set via \ + `MaxNominatorsCount` or `ElectionProvider`" + ); + Self::get_npos_voters_unbounded() + }, + }) } - fn targets(maybe_max_len: Option) -> data_provider::Result> { - let target_count = CounterForValidators::::get(); - - // We can't handle this case yet -- return an error. - if maybe_max_len.map_or(false, |max_len| target_count > max_len as u32) { - return Err("Target snapshot too big") - } - - Ok(Self::get_npos_targets()) + fn targets(maybe_max_size: Option) -> data_provider::Result> { + // On any reasonable chain, the validator candidates should be small enough for this to not + // need to truncate. Nonetheless, if it happens, we prefer truncating for now rather than + // returning an error. In the future, a second instance of the `SortedListProvider` should + // be used to sort validators as well in a cheap way. + Ok(match maybe_max_size { + Some(max_size) => Self::get_npos_targets_bounded(max_size), + None => { + log!( + warn, + "iterating over an unbounded number of npos targets, this might exhaust the \ + memory limits of the chain. Ensure proper limits are set via \ + `MaxValidatorsCount` or `ElectionProvider`" + ); + Self::get_npos_targets_unbounded() + }, + }) } fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber { @@ -1317,3 +1407,113 @@ impl SortedListProvider for UseNominatorsMap { } } } + +/// A static tracker for the snapshot of all voters. +/// +/// Computes the (SCALE) encoded byte length of a snapshot based on static rules, without any actual +/// encoding. +/// +/// ## Warning +/// +/// Make sure any change to SCALE is reflected here. +/// +/// ## Details +/// +/// The snapshot has a the form `Vec` where `Voter = (Account, u64, Vec)`. For each +/// voter added to the snapshot, [`register_voter`] should be called, with the number of votes +/// (length of the internal `Vec`). +/// +/// Whilst doing this, [`size`] will track the entire size of the `Vec`, except for the +/// length prefix of the outer `Vec`. To get the final size at any point, use +/// [`final_byte_size_of`]. +struct StaticSizeTracker { + size: usize, + _marker: sp_std::marker::PhantomData, +} + +impl StaticSizeTracker { + fn new() -> Self { + Self { size: 0, _marker: Default::default() } + } + + /// The length prefix of a vector with the given length. + #[inline] + fn length_prefix(length: usize) -> usize { + // TODO: scale codec could and should expose a public function for this that I can reuse. + match length { + 0..=63 => 1, + 64..=16383 => 2, + 16384..=1073741823 => 4, + // this arm almost always never happens. Although, it would be good to get rid of of it, + // for otherwise we could make this function const, which might enable further + // optimizations. + x @ _ => codec::Compact(x as u32).encoded_size(), + } + } + + /// Register a voter in `self` who has casted `votes`. + fn register_voter(&mut self, votes: usize) { + self.size = self.size.saturating_add(Self::voter_size(votes)) + } + + /// The byte size of a voter who casted `votes`. + fn voter_size(votes: usize) -> usize { + Self::length_prefix(votes) + // and each element + .saturating_add(votes * sp_std::mem::size_of::()) + // 1 vote-weight + .saturating_add(sp_std::mem::size_of::()) + // 1 voter account + .saturating_add(sp_std::mem::size_of::()) + } + + // Final size: size of all internal elements, plus the length prefix. + fn final_byte_size_of(&self, length: usize) -> usize { + self.size + Self::length_prefix(length) + } +} + +#[cfg(test)] +mod static_tracker { + use codec::Encode; + + use super::StaticSizeTracker; + + #[test] + fn len_prefix_works() { + let length_samples = + vec![0usize, 1, 62, 63, 64, 16383, 16384, 16385, 1073741822, 1073741823, 1073741824]; + + for s in length_samples { + // the encoded size of a vector of n bytes should be n + the length prefix + assert_eq!(vec![1u8; s].encoded_size(), StaticSizeTracker::::length_prefix(s) + s); + } + } + + #[test] + fn length_tracking_works() { + let mut voters: Vec<(u64, u64, Vec)> = vec![]; + let mut tracker = StaticSizeTracker::::new(); + + // initial state. + assert_eq!(voters.encoded_size(), tracker.final_byte_size_of(voters.len())); + + // add a bunch of stuff. + voters.push((1, 10, vec![1, 2, 3])); + tracker.register_voter(3); + assert_eq!(voters.encoded_size(), tracker.final_byte_size_of(voters.len())); + + voters.push((2, 20, vec![1, 3])); + tracker.register_voter(2); + assert_eq!(voters.encoded_size(), tracker.final_byte_size_of(voters.len())); + + voters.push((3, 30, vec![1])); + tracker.register_voter(1); + assert_eq!(voters.encoded_size(), tracker.final_byte_size_of(voters.len())); + + // unlikely to happen in reality, but anyways. + voters.push((4, 40, vec![])); + tracker.register_voter(0); + assert_eq!(voters.encoded_size(), tracker.final_byte_size_of(voters.len())); + } +} diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 8e97a90e07544..14938f353ab10 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -40,9 +40,9 @@ pub use impls::*; use crate::{ log, migrations, slashing, weights::WeightInfo, ActiveEraInfo, BalanceOf, EraIndex, EraPayout, - EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, Nominations, PositiveImbalanceOf, - Releases, RewardDestination, SessionInterface, StakingLedger, UnappliedSlash, UnlockChunk, - ValidatorPrefs, + EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, NominationQuota, Nominations, + PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakingLedger, + UnappliedSlash, UnlockChunk, ValidatorPrefs, }; pub const MAX_UNLOCKING_CHUNKS: usize = 32; @@ -89,9 +89,6 @@ pub mod pallet { DataProvider = Pallet, >; - /// Maximum number of nominations per nominator. - const MAX_NOMINATIONS: u32; - /// Tokens have been minted and are unused for validator-reward. /// See [Era payout](./index.html#era-payout). type RewardRemainder: OnUnbalanced>; @@ -150,16 +147,18 @@ pub mod pallet { /// the bags-list is not desired, [`impls::UseNominatorsMap`] is likely the desired option. type SortedListProvider: SortedListProvider; + /// Something that can dictate the number of nominations allowed per nominator. + type NominationQuota: NominationQuota>; + /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; } #[pallet::extra_constants] impl Pallet { - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - #[allow(non_snake_case)] - fn MaxNominations() -> u32 { - T::MAX_NOMINATIONS + #[pallet::constant_name(MaxNominations)] + fn max_nominations() -> u32 { + T::NominationQuota::ABSOLUTE_MAXIMUM } } @@ -973,12 +972,6 @@ pub mod pallet { /// Effects will be felt at the beginning of the next era. /// /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// # - /// - The transaction's complexity is proportional to the size of `targets` (N) - /// which is capped at CompactAssignments::LIMIT (MAX_NOMINATIONS). - /// - Both the reads and writes follow a similar pattern. - /// # #[pallet::weight(T::WeightInfo::nominate(targets.len() as u32))] pub fn nominate( origin: OriginFor, @@ -1004,7 +997,14 @@ pub mod pallet { } ensure!(!targets.is_empty(), Error::::EmptyTargets); - ensure!(targets.len() <= T::MAX_NOMINATIONS as usize, Error::::TooManyTargets); + + ensure!( + targets.len() as u32 <= + T::NominationQuota::nomination_quota(Self::slashable_balance_of( + &ledger.stash + )), + Error::::TooManyTargets + ); let old = Nominators::::get(stash).map_or_else(Vec::new, |x| x.targets); @@ -1595,6 +1595,34 @@ pub mod pallet { Self::chill_stash(&stash); Ok(()) } + + /// Update a nominator's nomination, if some of their nomination targets have been slashed. + /// + /// These votes are effectively being stripped in the election process. Removing them via + /// this extrinsic will be a pre-emptive cleanup that benefits the chain. + /// + /// Returns the transaction fee upon successful execution. + #[pallet::weight(0)] + pub fn update_slashed_nominator( + origin: OriginFor, + stash: T::AccountId, + ) -> DispatchResultWithPostInfo { + let _ = ensure_signed(origin); + Nominators::::try_mutate(stash, |maybe_nomination| { + if let Some(Nominations { targets, submitted_in, suppressed: _ }) = maybe_nomination + { + let initial_len = targets.len(); + targets.retain(|v| { + SlashingSpans::::get(v) + .map_or(true, |spans| *submitted_in >= spans.last_nonzero_slash()) + }); + if initial_len != targets.len() { + return Ok(Pays::No.into()) + } + } + Err("not slashed".into()) + }) + } } } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index d6d92d5bd57fc..597166058f00a 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -1966,8 +1966,8 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider() { assert_eq!( supports, vec![ - (21, Support { total: 1800, voters: vec![(21, 1000), (1, 400), (3, 400)] }), - (31, Support { total: 2200, voters: vec![(31, 1000), (1, 600), (3, 600)] }) + (21, Support { total: 1800, voters: vec![(1, 400), (3, 400), (21, 1000)] }), + (31, Support { total: 2200, voters: vec![(1, 600), (3, 600), (31, 1000)] }) ], ); }); @@ -2010,8 +2010,8 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider_elected() { assert_eq!( supports, vec![ - (11, Support { total: 1500, voters: vec![(11, 1000), (1, 500)] }), - (21, Support { total: 2500, voters: vec![(21, 1000), (1, 500), (3, 1000)] }) + (11, Support { total: 1500, voters: vec![(1, 500), (11, 1000)] }), + (21, Support { total: 2500, voters: vec![(1, 500), (3, 1000), (21, 1000)] }) ], ); }); @@ -3982,18 +3982,6 @@ mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; - #[test] - fn targets_2sec_block() { - let mut validators = 1000; - while ::WeightInfo::get_npos_targets(validators) < - 2 * frame_support::weights::constants::WEIGHT_PER_SECOND - { - validators += 1; - } - - println!("Can create a snapshot of {} validators in 2sec block", validators); - } - #[test] fn voters_2sec_block() { // we assume a network only wants up to 1000 validators in most cases, thus having 2000 @@ -4003,8 +3991,11 @@ mod election_data_provider { let slashing_spans = validators; let mut nominators = 1000; - while ::WeightInfo::get_npos_voters(validators, nominators, slashing_spans) < - 2 * frame_support::weights::constants::WEIGHT_PER_SECOND + while ::WeightInfo::get_npos_voters_bounded( + validators, + nominators, + slashing_spans, + ) < 2 * frame_support::weights::constants::WEIGHT_PER_SECOND { nominators += 1; } @@ -4070,10 +4061,20 @@ mod election_data_provider { } #[test] - fn respects_snapshot_len_limits() { + fn get_npos_voters_works() { ExtBuilder::default() .set_status(41, StakerStatus::Validator) .build_and_execute(|| { + let limit_for = |c| { + Some( + Staking::voters(None) + .unwrap() + .into_iter() + .take(c) + .collect::>() + .encoded_size(), + ) + }; // sum of all nominators who'd be voters (1), plus the self-votes (4). assert_eq!( ::SortedListProvider::count() + @@ -4081,103 +4082,99 @@ mod election_data_provider { 5 ); + // unbounded: + assert_eq!( + Staking::voters(None).unwrap(), + vec![ + (101, 500, vec![11, 21]), // 8 + 8 + (8 * 2) + 1 = 33 + (31, 500, vec![31]), // 8 + 8 + 8 + 1 = 25 + (41, 1000, vec![41]), + (21, 1000, vec![21]), + (11, 1000, vec![11]), + ] + ); + // if limits is less.. - assert_eq!(Staking::voters(Some(1)).unwrap().len(), 1); + // let's check one of the manually for some mental practice + assert_eq!(limit_for(2).unwrap(), 33 + 25 + 1); + assert_eq!(Staking::voters(limit_for(2)).unwrap().len(), 2); + + // edge-case: we have enough size only for all validators, and none of the + // nominators. + let limit_validators = limit_for(>::iter().count()); + assert_eq!(Staking::voters(limit_validators).unwrap().len(), 4); // if limit is equal.. - assert_eq!(Staking::voters(Some(5)).unwrap().len(), 5); + assert_eq!(Staking::voters(limit_for(5)).unwrap().len(), 5); // if limit is more. - assert_eq!(Staking::voters(Some(55)).unwrap().len(), 5); - - // if target limit is more.. - assert_eq!(Staking::targets(Some(6)).unwrap().len(), 4); - assert_eq!(Staking::targets(Some(4)).unwrap().len(), 4); - - // if target limit is less, then we return an error. - assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); + assert_eq!(Staking::voters(Some(limit_for(5).unwrap() * 2)).unwrap().len(), 5); }); } #[test] - fn only_iterates_max_2_times_nominators_quota() { + fn respects_targets_snapshot_len_limits() { ExtBuilder::default() - .nominate(true) // add nominator 101, who nominates [11, 21] - // the other nominators only nominate 21 - .add_staker(61, 60, 2_000, StakerStatus::::Nominator(vec![21])) - .add_staker(71, 70, 2_000, StakerStatus::::Nominator(vec![21])) - .add_staker(81, 80, 2_000, StakerStatus::::Nominator(vec![21])) + .set_status(41, StakerStatus::Validator) .build_and_execute(|| { - // given our nominators ordered by stake, - assert_eq!( - ::SortedListProvider::iter().collect::>(), - vec![61, 71, 81, 101] - ); - - // and total voters - assert_eq!( - ::SortedListProvider::count() + - >::iter().count() as u32, - 7 - ); + let limit_for = |c| { + Some( + Staking::targets(None) + .unwrap() + .into_iter() + .take(c) + .collect::>() + .encoded_size(), + ) + }; + + // all targets: + assert_eq!(>::iter().count() as u32, 4); + + // unbounded: + assert_eq!(Staking::targets(None).unwrap().len(), 4); - // roll to session 5 - run_to_block(25); - - // slash 21, the only validator nominated by our first 3 nominators - add_slash(&21); + // if target limit is more.. + assert_eq!(Staking::targets(limit_for(8)).unwrap().len(), 4); + assert_eq!(Staking::targets(limit_for(4)).unwrap().len(), 4); - // we take 4 voters: 2 validators and 2 nominators (so nominators quota = 2) - assert_eq!( - Staking::voters(Some(3)) - .unwrap() - .iter() - .map(|(stash, _, _)| stash) - .copied() - .collect::>(), - vec![31, 11], // 2 validators, but no nominators because we hit the quota - ); - }); + // if target limit is less.. + assert_eq!(Staking::targets(limit_for(3)).unwrap().len(), 3); + assert_eq!(Staking::targets(limit_for(1)).unwrap().len(), 1); + }) } - // Even if some of the higher staked nominators are slashed, we still get up to max len voters - // by adding more lower staked nominators. In other words, we assert that we keep on adding - // valid nominators until we reach max len voters; which is opposed to simply stopping after we - // have iterated max len voters, but not adding all of them to voters due to some nominators not - // having valid targets. + // Even if some of the higher staked nominators are slashed, we don't get up to max len voters + // by adding more lower staked nominators. #[test] fn get_max_len_voters_even_if_some_nominators_are_slashed() { ExtBuilder::default() .nominate(true) // add nominator 101, who nominates [11, 21] .add_staker(61, 60, 20, StakerStatus::::Nominator(vec![21])) - // 61 only nominates validator 21 ^^ .add_staker(71, 70, 10, StakerStatus::::Nominator(vec![11, 21])) .build_and_execute(|| { - // given our nominators ordered by stake, - assert_eq!( - ::SortedListProvider::iter().collect::>(), - vec![101, 61, 71] - ); - - // and total voters - assert_eq!( - ::SortedListProvider::count() + - >::iter().count() as u32, - 6 - ); + // given + assert_eq!(validator_ids(), vec![31, 21, 11]); + assert_eq!(nominator_ids(), vec![101, 71, 61]); // we take 5 voters + let limit_5 = Staking::voters(None) + .unwrap() + .into_iter() + .take(5) + .collect::>() + .encoded_size(); assert_eq!( - Staking::voters(Some(5)) + Staking::voters(Some(limit_5)) .unwrap() .iter() .map(|(stash, _, _)| stash) .copied() .collect::>(), - // then vec![ - 31, 21, 11, // 3 nominators - 101, 61 // 2 validators, and 71 is excluded + 31, 21, 11, // all validators are included + // and 2 nominator, and 71 is excluded, because it has less stake. + 101, 61, ], ); @@ -4188,8 +4185,14 @@ mod election_data_provider { add_slash(&21); // we take 4 voters + let limit_4 = Staking::voters(None) + .unwrap() + .into_iter() + .take(4) + .collect::>() + .encoded_size(); assert_eq!( - Staking::voters(Some(4)) + Staking::voters(Some(limit_4)) .unwrap() .iter() .map(|(stash, _, _)| stash) @@ -4197,7 +4200,9 @@ mod election_data_provider { .collect::>(), vec![ 31, 11, // 2 validators (21 was slashed) - 101, 71 // 2 nominators, excluding 61 + 101, + /* 61 is skipped, since it has a slash now. 71 is not included, because + * we still count that as part of the byte `size_limit`. */ ], ); }); @@ -4548,6 +4553,30 @@ fn capped_stakers_works() { }) } +#[test] +fn update_slashed_nominator_works() { + ExtBuilder::default().nominate(true).build_and_execute(|| { + start_active_era(1); + + // 101 nominates 11 by default. + assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); + + // can't update them at all. + assert_noop!(Staking::update_slashed_nominator(Origin::signed(31), 101), "not slashed"); + + // but it gets slashed. + add_slash(&11); + start_active_era(2); + + // 11 is cleaned now. + assert_ok!(Staking::update_slashed_nominator(Origin::signed(31), 101)); + assert_eq!(Staking::nominators(101).unwrap().targets, vec![21]); + + // can't repeat either + assert_noop!(Staking::update_slashed_nominator(Origin::signed(31), 101), "not slashed"); + }); +} + mod sorted_list_provider { use super::*; use frame_election_provider_support::SortedListProvider; diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 32c8dc80da158..bb9ec0b853d2d 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -69,8 +69,10 @@ pub trait WeightInfo { fn set_history_depth(e: u32, ) -> Weight; fn reap_stash(s: u32, ) -> Weight; fn new_era(v: u32, n: u32, ) -> Weight; - fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight; - fn get_npos_targets(v: u32, ) -> Weight; + fn get_npos_voters_bounded(v: u32, n: u32, s: u32, ) -> Weight; + fn get_npos_voters_unbounded(v: u32, n: u32, s: u32, ) -> Weight; + fn get_npos_targets_bounded(v: u32, ) -> Weight; + fn get_npos_targets_unbounded(v: u32, ) -> Weight; fn set_staking_limits() -> Weight; fn chill_other() -> Weight; } @@ -397,7 +399,7 @@ impl WeightInfo for SubstrateWeight { // Storage: BagsList ListBags (r:200 w:0) // Storage: BagsList ListNodes (r:1000 w:0) // Storage: Staking Nominators (r:1000 w:0) - fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { + fn get_npos_voters_bounded(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) // Standard Error: 91_000 .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) @@ -410,8 +412,38 @@ impl WeightInfo for SubstrateWeight { .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } + // Storage: Staking CounterForNominators (r:1 w:0) + // Storage: Staking CounterForValidators (r:1 w:0) + // Storage: Staking Validators (r:501 w:0) + // Storage: Staking Bonded (r:1500 w:0) + // Storage: Staking Ledger (r:1500 w:0) + // Storage: Staking SlashingSpans (r:21 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:1000 w:0) + // Storage: Staking Nominators (r:1000 w:0) + fn get_npos_voters_unbounded(v: u32, n: u32, s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 91_000 + .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 91_000 + .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_122_000 + .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(204 as Weight)) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Staking Validators (r:501 w:0) + fn get_npos_targets_bounded(v: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 34_000 + .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) + } // Storage: Staking Validators (r:501 w:0) - fn get_npos_targets(v: u32, ) -> Weight { + fn get_npos_targets_unbounded(v: u32, ) -> Weight { (0 as Weight) // Standard Error: 34_000 .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) @@ -765,7 +797,7 @@ impl WeightInfo for () { // Storage: BagsList ListBags (r:200 w:0) // Storage: BagsList ListNodes (r:1000 w:0) // Storage: Staking Nominators (r:1000 w:0) - fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { + fn get_npos_voters_bounded(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) // Standard Error: 91_000 .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) @@ -778,8 +810,38 @@ impl WeightInfo for () { .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } + // Storage: Staking CounterForNominators (r:1 w:0) + // Storage: Staking CounterForValidators (r:1 w:0) + // Storage: Staking Validators (r:501 w:0) + // Storage: Staking Bonded (r:1500 w:0) + // Storage: Staking Ledger (r:1500 w:0) + // Storage: Staking SlashingSpans (r:21 w:0) + // Storage: BagsList ListBags (r:200 w:0) + // Storage: BagsList ListNodes (r:1000 w:0) + // Storage: Staking Nominators (r:1000 w:0) + fn get_npos_voters_unbounded(v: u32, n: u32, s: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 91_000 + .saturating_add((26_605_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 91_000 + .saturating_add((31_481_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_122_000 + .saturating_add((16_672_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(204 as Weight)) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) + } + // Storage: Staking Validators (r:501 w:0) + fn get_npos_targets_bounded(v: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 34_000 + .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) + } // Storage: Staking Validators (r:501 w:0) - fn get_npos_targets(v: u32, ) -> Weight { + fn get_npos_targets_unbounded(v: u32, ) -> Weight { (0 as Weight) // Standard Error: 34_000 .saturating_add((10_558_000 as Weight).saturating_mul(v as Weight)) From f9a27f1b7809434d6bf653aea1021a4a76ee26aa Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 15 Nov 2021 15:10:35 +0100 Subject: [PATCH 2/7] new version with dual measurement --- .../election-provider-multi-phase/src/lib.rs | 41 +++-- .../election-provider-multi-phase/src/mock.rs | 44 ++--- frame/election-provider-support/src/lib.rs | 85 ++++++++- .../election-provider-support/src/onchain.rs | 14 +- frame/staking/Cargo.toml | 2 + frame/staking/src/pallet/impls.rs | 161 +++++++++++------- frame/staking/src/tests.rs | 46 ++--- 7 files changed, 251 insertions(+), 142 deletions(-) diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 57d868caeaeb1..7976701eb74ad 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -230,7 +230,7 @@ #![cfg_attr(not(feature = "std"), no_std)] use codec::{Decode, Encode}; -use frame_election_provider_support::{ElectionDataProvider, ElectionProvider}; +use frame_election_provider_support::{ElectionDataProvider, ElectionProvider, SnapshotBounds}; use frame_support::{ dispatch::DispatchResultWithPostInfo, ensure, @@ -640,20 +640,16 @@ pub mod pallet { #[pallet::constant] type SignedDepositWeight: Get>; - /// The maximum byte-size of voters to put in the snapshot. - /// - /// At the moment, snapshots are only over a single block, but once multi-block elections - /// are introduced they will take place over multiple blocks. + /// The bound on the amount of voters to put in the snapshot, per block. #[pallet::constant] - type VoterSnapshotSizePerBlock: Get; + type VoterSnapshotBounds: Get; - /// The maximum byte size of targets to put in the snapshot. + /// The bound on the amount of targets to put in the snapshot, per block. /// - /// The target snapshot is assumed to always fit in one block, and happens next to voter - /// snapshot. In essence, in a single block, `TargetSnapshotSize + - /// VoterSnapshotSizePerBlock` bytes of data are present in a snapshot. + /// Note that the target snapshot happens next to voter snapshot. In essence, in a single + /// block, `TargetSnapshotSize + VoterSnapshotBounds` must not exhaust. #[pallet::constant] - type TargetSnapshotSize: Get; + type TargetSnapshotBounds: Get; /// Handler for the slashed deposits. type SlashHandler: OnUnbalanced>; @@ -840,8 +836,9 @@ pub mod pallet { // ---------------------------- // maximum size of a snapshot should not exceed half the size of our allocator limit. assert!( - T::VoterSnapshotSizePerBlock::get().saturating_add(T::TargetSnapshotSize::get()) < - sp_core::MAX_POSSIBLE_ALLOCATION / 2 - 1 + T::VoterSnapshotBounds::get().count_bound().unwrap_or_default().saturating_add( + T::TargetSnapshotBounds::get().count_bound().unwrap_or_default() + ) < sp_core::MAX_POSSIBLE_ALLOCATION as usize / 2 - 1 ); } } @@ -1305,20 +1302,22 @@ impl Pallet { /// Extracted for easier weight calculation. fn create_snapshot_external( ) -> Result<(Vec, Vec>, u32), ElectionError> { - let target_limit = Some(T::TargetSnapshotSize::get() as usize); - let voter_limit = Some(T::VoterSnapshotSizePerBlock::get() as usize); + let target_bound = T::TargetSnapshotBounds::get(); + let voter_bound = T::VoterSnapshotBounds::get(); let targets = - T::DataProvider::targets(target_limit).map_err(ElectionError::DataProvider)?; - let voters = T::DataProvider::voters(voter_limit).map_err(ElectionError::DataProvider)?; + T::DataProvider::targets(target_bound).map_err(ElectionError::DataProvider)?; + let voters = T::DataProvider::voters(voter_bound).map_err(ElectionError::DataProvider)?; let desired_targets = T::DataProvider::desired_targets().map_err(ElectionError::DataProvider)?; // Defensive-only. - if voter_limit.map_or(false, |l| voters.len() > l) { - debug_assert!(false, "Snapshot limit has not been respected."); - return Err(ElectionError::DataProvider("Snapshot too big for submission.")) - } + debug_assert!( + !voter_bound.exhausted(|| voters.encoded_size() as u32, || voters.len() as u32) + ); + debug_assert!( + !target_bound.exhausted(|| targets.encoded_size() as u32, || targets.len() as u32) + ); Ok((targets, voters, desired_targets)) } diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index c3d13a65b494a..fa9f9e41b63b2 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -18,7 +18,7 @@ use super::*; use crate as multi_phase; use frame_election_provider_support::{ - data_provider, onchain, ElectionDataProvider, SequentialPhragmen, + data_provider, onchain, ElectionDataProvider, SequentialPhragmen, SnapshotBounds, }; pub use frame_support::{assert_noop, assert_ok}; use frame_support::{parameter_types, traits::Hooks, weights::Weight}; @@ -37,7 +37,7 @@ use sp_npos_elections::{ }; use sp_runtime::{ testing::Header, - traits::{BlakeTwo256, IdentityLookup}, + traits::{BlakeTwo256, Bounded, IdentityLookup}, PerU16, }; use std::sync::Arc; @@ -268,8 +268,8 @@ parameter_types! { pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxLength: u32 = 256; pub static MockWeightInfo: bool = false; - pub static VoterSnapshotSizePerBlock: u32 = 4 * 1024 * 1024; - pub static TargetSnapshotSize: u32 = 1 * 1024 * 1024; + pub static VoterSnapshotBounds: SnapshotBounds = SnapshotBounds::new_unbounded(); + pub static TargetSnapshotBounds: SnapshotBounds = SnapshotBounds::new_unbounded(); // TODO: we want a test to see: // - how many nominators we can create in a normal distribution, for a runtime @@ -412,8 +412,8 @@ impl crate::Config for Runtime { type Fallback = MockFallback; type ForceOrigin = frame_system::EnsureRoot; type Solution = TestNposSolution; - type VoterSnapshotSizePerBlock = VoterSnapshotSizePerBlock; - type TargetSnapshotSize = TargetSnapshotSize; + type VoterSnapshotBounds = VoterSnapshotBounds; + type TargetSnapshotBounds = TargetSnapshotBounds; type Solver = SequentialPhragmen, Balancing>; } @@ -433,10 +433,12 @@ pub struct ExtBuilder {} pub struct StakingMock; impl ElectionDataProvider for StakingMock { const MAXIMUM_VOTES_PER_VOTER: u32 = ::LIMIT as u32; - fn targets(maybe_max_size: Option) -> data_provider::Result> { + fn targets(bounds: SnapshotBounds) -> data_provider::Result> { let targets = Targets::get(); - if maybe_max_size.map_or(false, |max_len| targets.encoded_size() > max_len) { + if bounds.size_exhausted(|| targets.encoded_size() as u32) || + bounds.count_exhausted(|| targets.len() as u32) + { return Err("Targets too big") } @@ -444,19 +446,17 @@ impl ElectionDataProvider for StakingMock { } fn voters( - maybe_max_size: Option, + bounds: SnapshotBounds, ) -> data_provider::Result)>> { let mut voters = Voters::get(); - if let Some(max_len) = maybe_max_size { - while voters.encoded_size() > max_len && !voters.is_empty() { - log::trace!( - target: crate::LOG_TARGET, - "staking-mock: truncating length to {}", - voters.len() - 1 - ); - let _ = voters.pop(); - } + while bounds.size_exhausted(|| voters.encoded_size() as u32) && !voters.is_empty() { + let _ = voters.pop(); + } + if bounds.count_exhausted(|| voters.len() as u32) { + voters.truncate( + bounds.count_bound().map(|b| b as usize).unwrap_or_else(Bounded::max_value), + ); } Ok(voters) @@ -516,7 +516,7 @@ mod staking_mock { // the `StakingMock` is designed such that if it has too many targets, it simply fails. ExtBuilder::default().build_and_execute(|| { // 1 byte won't allow for anything. - TargetSnapshotSize::set(1); + TargetSnapshotBounds::set(SnapshotBounds::new_count(1)); // Signed phase failed to open. roll_to(15); @@ -537,7 +537,7 @@ mod staking_mock { fn snapshot_too_big_failure_no_fallback() { // .. and if the backup mode is nothing, we go into the emergency mode.. ExtBuilder::default().onchain_fallback(false).build_and_execute(|| { - TargetSnapshotSize::set(1); + TargetSnapshotBounds::set(SnapshotBounds::new_count(1)); // Signed phase failed to open. roll_to(15); @@ -563,12 +563,12 @@ mod staking_mock { assert_eq!(crate::mock::Voters::get().len(), 8); // the byte-size of taking the first two voters. - let limit_2 = crate::mock::Voters::get() + let size_limit_2 = crate::mock::Voters::get() .into_iter() .take(2) .collect::>() .encoded_size() as u32; - crate::mock::VoterSnapshotSizePerBlock::set(limit_2); + VoterSnapshotBounds::set(SnapshotBounds::new_size(size_limit_2)); // Signed phase opens just fine. roll_to(15); diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 387a6edee447e..47ea4a7f997bb 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -103,12 +103,12 @@ //! fn desired_targets() -> data_provider::Result { //! Ok(1) //! } -//! fn voters(maybe_max_len: Option) +//! fn voters(_bounds: SnapshotBounds) //! -> data_provider::Result)>> //! { //! Ok(Default::default()) //! } -//! fn targets(maybe_max_len: Option) -> data_provider::Result> { +//! fn targets(_bounds: SnapshotBounds) -> data_provider::Result> { //! Ok(vec![10, 20, 30]) //! } //! fn next_election_prediction(now: BlockNumber) -> BlockNumber { @@ -132,7 +132,7 @@ //! type DataProvider = T::DataProvider; //! //! fn elect() -> Result, Self::Error> { -//! Self::DataProvider::targets(None) +//! Self::DataProvider::targets(SnapshotBounds::new_unbounded()) //! .map_err(|_| "failed to elect") //! .map(|t| vec![(t[0], Support::default())]) //! } @@ -161,7 +161,8 @@ #![cfg_attr(not(feature = "std"), no_std)] pub mod onchain; -use frame_support::traits::Get; +use codec::{Decode, Encode}; +use frame_support::{traits::Get, RuntimeDebug}; use sp_std::{fmt::Debug, prelude::*}; /// Re-export some type as they are used in the interface. @@ -191,7 +192,7 @@ pub trait ElectionDataProvider { /// /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. - fn targets(maybe_max_size: Option) -> data_provider::Result>; + fn targets(bounds: SnapshotBounds) -> data_provider::Result>; /// All possible voters for the election. /// @@ -203,7 +204,7 @@ pub trait ElectionDataProvider { /// This should be implemented as a self-weighing function. The implementor should register its /// appropriate weight at the end of execution with the system pallet directly. fn voters( - maybe_max_size: Option, + bounds: SnapshotBounds, ) -> data_provider::Result)>>; /// The number of targets to elect. @@ -252,11 +253,11 @@ pub trait ElectionDataProvider { #[cfg(feature = "std")] impl ElectionDataProvider for () { const MAXIMUM_VOTES_PER_VOTER: u32 = 0; - fn targets(_: Option) -> data_provider::Result> { + fn targets(_: SnapshotBounds) -> data_provider::Result> { Ok(Default::default()) } fn voters( - _: Option, + _: SnapshotBounds, ) -> data_provider::Result)>> { Ok(Default::default()) } @@ -433,3 +434,71 @@ impl< sp_npos_elections::phragmms(winners, targets, voters, Balancing::get()) } } + +/// The limits imposed on a snapshot, either voters or targets. +#[derive(Clone, Copy, RuntimeDebug, scale_info::TypeInfo, Encode, Decode)] +pub struct SnapshotBounds { + size: Option, + count: Option, +} + +impl SnapshotBounds { + pub fn new_size(size: u32) -> Self { + Self { size: Some(size), count: None } + } + + pub fn new_count(count: u32) -> Self { + Self { count: Some(count), size: None } + } + + pub fn new(count: u32, size: u32) -> Self { + Self { count: Some(count), size: Some(size) } + } + + pub fn new_unbounded() -> Self { + Self { size: None, count: None } + } + + pub fn size_exhausted(&self, given_size: impl FnOnce() -> u32) -> bool { + self.size.map_or(false, |size| given_size() > size) + } + + pub fn count_exhausted(&self, given_count: impl FnOnce() -> u32) -> bool { + self.count.map_or(false, |count| given_count() > count) + } + + // TODO: don't like the signature, don't like the u32 a raw type that can get confused. Luckily + // it is not used all that much. + pub fn exhausted( + &self, + given_size: impl FnOnce() -> u32, + given_count: impl FnOnce() -> u32, + ) -> bool { + self.count_exhausted(given_count) || self.size_exhausted(given_size) + } + + pub fn size_bound(&self) -> Option { + self.size.map(|b| b as usize) + } + + pub fn count_bound(&self) -> Option { + self.count.map(|b| b as usize) + } + + pub fn is_unbounded(&self) -> bool { + self.size.is_none() && self.count.is_none() + } + + pub fn is_bounded(&self) -> bool { + !self.is_unbounded() + } + + pub fn predict_capacity(&self, item_size: usize) -> Option { + match (self.size.map(|x| x as usize), self.count.map(|x| x as usize)) { + (Some(max_size), Some(max_count)) => Some(max_count.min(max_size / item_size.max(1))), + (Some(max_size), None) => Some(max_size / item_size.max(1)), + (None, Some(max_count)) => Some(max_count), + (None, None) => None, + } + } +} diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index fb1ccfdfe2566..6b1fda77b9289 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -17,7 +17,7 @@ //! An implementation of [`ElectionProvider`] that does an on-chain sequential phragmen. -use crate::{ElectionDataProvider, ElectionProvider}; +use crate::{ElectionDataProvider, ElectionProvider, SnapshotBounds}; use frame_support::{traits::Get, weights::DispatchClass}; use sp_npos_elections::*; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData, prelude::*}; @@ -70,8 +70,10 @@ impl ElectionProvider for OnChainSequen type DataProvider = T::DataProvider; fn elect() -> Result, Self::Error> { - let voters = Self::DataProvider::voters(None).map_err(Error::DataProvider)?; - let targets = Self::DataProvider::targets(None).map_err(Error::DataProvider)?; + let voters = Self::DataProvider::voters(SnapshotBounds::new_unbounded()) + .map_err(Error::DataProvider)?; + let targets = Self::DataProvider::targets(SnapshotBounds::new_unbounded()) + .map_err(Error::DataProvider)?; let desired_targets = Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; let stake_map: BTreeMap = voters @@ -156,18 +158,18 @@ mod tests { mod mock_data_provider { use super::*; - use crate::data_provider; + use crate::{data_provider, SnapshotBounds}; pub struct DataProvider; impl ElectionDataProvider for DataProvider { const MAXIMUM_VOTES_PER_VOTER: u32 = 2; fn voters( - _: Option, + _: SnapshotBounds, ) -> data_provider::Result)>> { Ok(vec![(1, 10, vec![10, 20]), (2, 20, vec![30, 20]), (3, 30, vec![10, 30])]) } - fn targets(_: Option) -> data_provider::Result> { + fn targets(_: SnapshotBounds) -> data_provider::Result> { Ok(vec![10, 20, 30]) } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index d9461ab454f39..913a290109f89 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -20,6 +20,7 @@ codec = { package = "parity-scale-codec", version = "2.0.0", default-features = scale-info = { version = "1.0", default-features = false, features = ["derive"] } sp-std = { version = "4.0.0-dev", default-features = false, path = "../../primitives/std" } sp-io = { version = "4.0.0-dev", default-features = false, path = "../../primitives/io" } +sp-core = { version = "4.0.0-dev", default-features = false, path = "../../primitives/core" } sp-runtime = { version = "4.0.0-dev", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" } @@ -57,6 +58,7 @@ std = [ "scale-info/std", "sp-std/std", "sp-io/std", + "sp-core/std", "frame-support/std", "sp-runtime/std", "sp-staking/std", diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index ec639406c210a..02474312af1cf 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -18,8 +18,8 @@ //! Implementations for the Staking FRAME Pallet. use frame_election_provider_support::{ - data_provider, ElectionDataProvider, ElectionProvider, SortedListProvider, Supports, - VoteWeight, VoteWeightProvider, + data_provider, ElectionDataProvider, ElectionProvider, SnapshotBounds, SortedListProvider, + Supports, VoteWeight, VoteWeightProvider, }; use frame_support::{ pallet_prelude::*, @@ -705,7 +705,7 @@ impl Pallet { /// Get all of the voters that are eligible for the next npos election. /// - /// `max_size` imposes a cap on the byte-size of the entire voters returned. + /// `bounds` imposes a cap on the count and byte-size of the entire voters returned. /// /// As of now, first all the validator are included in no particular order, then remainder is /// taken from the nominators, as returned by [`Config::SortedListProvider`]. @@ -715,35 +715,58 @@ impl Pallet { /// ### Slashing /// /// All nominations that have been submitted before the last non-zero slash of the validator are - /// auto-chilled, and they DO count towards the limit imposed by `maybe_max_size`. - /// - /// In essence, this implementation ensures that no more than `max_size` bytes worth of npos - /// voters are READ from storage, which subsequently ensures that the returning value cannot be - /// more than `max_size`. + /// auto-chilled, and they **DO** count towards the limit imposed by `bounds`. To prevent this + /// from getting in the way, [`update_slashed_nominator`] can be used to clean these stale + /// nominations. pub fn get_npos_voters_bounded( - max_size: usize, + bounds: SnapshotBounds, ) -> Vec<(T::AccountId, VoteWeight, Vec)> { let mut tracker = StaticSizeTracker::::new(); - let mut voters = Vec::<_>::with_capacity( - max_size / - StaticSizeTracker::::voter_size( - T::NominationQuota::ABSOLUTE_MAXIMUM as usize, - ), - ); + let mut voters = if let Some(capacity) = + bounds.predict_capacity(StaticSizeTracker::::voter_size( + T::NominationQuota::ABSOLUTE_MAXIMUM as usize, + )) { + Vec::with_capacity(capacity.min(sp_core::MAX_POSSIBLE_ALLOCATION as usize / 2)) + } else { + Vec::new() + }; + + // register a voter with `votes` with regards to bounds. + let add_voter = |tracker_ref: &mut StaticSizeTracker, votes: usize| { + if let Some(_) = bounds.size_bound() { + tracker_ref.register_voter(votes) + } else if let Some(_) = bounds.count_bound() { + // nothing needed, voters.len() is itself a representation. + } + }; + + // check if adding one more voter will exhaust any of the bounds. + let next_will_exhaust = |tracker_ref: &StaticSizeTracker, + voters_ref: &Vec<_>| match ( + bounds.size_bound(), + bounds.count_bound(), + ) { + (Some(max_size), Some(max_count)) => + tracker_ref.final_byte_size_of(voters_ref.len() + 1) > max_size || + voters_ref.len() + 1 > max_count, + (Some(max_size), None) => + tracker_ref.final_byte_size_of(voters_ref.len() + 1) > max_size, + (None, Some(max_count)) => voters_ref.len() + 1 > max_count, + (None, None) => false, + }; // first, grab all validators in no particular order. In most cases, all of them should fit // anyway. for (validator, _) in >::iter() { - // Append self vote. let self_vote = (validator.clone(), Self::weight_of(&validator), vec![validator.clone()]); - tracker.register_voter(1usize); - if tracker.final_byte_size_of(voters.len() + 1) > max_size { + add_voter(&mut tracker, 1); + if next_will_exhaust(&tracker, &voters) { log!( warn, - "stopped iterating over validators' self-vote at {} to cap the size at {}. This should probably never happen", + "stopped iterating over validators' self-vote at {} due to bound {:?}", voters.len(), - max_size, + bounds, ); break } @@ -759,9 +782,11 @@ impl Pallet { if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = >::get(&nominator) { - // IMPORTANT: we track the size and potentially break out right here. - tracker.register_voter(targets.len()); - if tracker.final_byte_size_of(voters.len() + 1) > max_size { + // IMPORTANT: we track the size and potentially break out right here. This ensures + // that votes that are invalid will also affect the snapshot bounds. Chain operators + // should ensure `update_slashed_nominator` is used to eliminate the need for this. + add_voter(&mut tracker, targets.len()); + if next_will_exhaust(&tracker, &voters) { break } @@ -784,21 +809,22 @@ impl Pallet { nominators_taken as u32, slashing_spans.len() as u32, )); - debug_assert!(voters.encoded_size() <= max_size); + debug_assert!(!bounds.exhausted(|| voters.encoded_size() as u32, || voters.len() as u32)); log!( info, - "generated {} npos voters, {} from validators and {} nominators, with size limit {}", + "generated {} npos voters, {} from validators and {} nominators, with bound {:?}", voters.len(), validators_taken, nominators_taken, - max_size, + bounds, ); voters } /// Get the list of targets (validators) that are eligible for the next npos election. - // This function is self-weighing as [`DispatchClass::Mandatory`]. + /// + /// This function is self-weighing as [`DispatchClass::Mandatory`]. /// /// # Warning /// @@ -813,19 +839,35 @@ impl Pallet { /// Get the list of targets (validators) that are eligible for the next npos election. /// - /// `max_size` imposes a cap on the byte-size of the entire targets returned. + /// `bounds` imposes a cap on the count and byte-size of the entire targets returned. /// /// This function is self-weighing as [`DispatchClass::Mandatory`]. - pub fn get_npos_targets_bounded(max_size: usize) -> Vec { + pub fn get_npos_targets_bounded(bounds: SnapshotBounds) -> Vec { let mut internal_size: usize = Zero::zero(); - let mut targets: Vec = - Vec::with_capacity(max_size / sp_std::mem::size_of::()); + let mut targets: Vec = if let Some(capacity) = + bounds.predict_capacity(sp_std::mem::size_of::()) + { + Vec::with_capacity(capacity.min(sp_core::MAX_POSSIBLE_ALLOCATION as usize / 2)) + } else { + Vec::new() + }; + + let next_will_exhaust = + |new_final_size, new_count| match (bounds.size_bound(), bounds.count_bound()) { + (Some(max_size), Some(max_count)) => + new_final_size > max_size || new_count > max_count, + (Some(max_size), None) => new_final_size > max_size, + (None, Some(max_count)) => new_count > max_count, + (None, None) => false, + }; for (next, _) in Validators::::iter() { + // TODO: rather sub-optimal, we should not need to track size if it is not bounded. let new_internal_size = internal_size + sp_std::mem::size_of::(); let new_final_size = new_internal_size + StaticSizeTracker::::length_prefix(targets.len() + 1); - if new_final_size > max_size { + let new_count = targets.len() + 1; + if next_will_exhaust(new_final_size, new_count) { // we've had enough break } @@ -836,14 +878,9 @@ impl Pallet { } Self::register_weight(T::WeightInfo::get_npos_targets_bounded(targets.len() as u32)); - debug_assert!( - targets.encoded_size() <= max_size, - "encoded size: {}, max_size: {}", - targets.encoded_size(), - max_size - ); + debug_assert!(!bounds.exhausted(|| targets.encoded_size() as u32, || targets.len() as u32)); - log!(info, "generated {} npos targets, with size limit {}", targets.len(), max_size); + log!(info, "generated {} npos targets, with bounds limit {:?}", targets.len(), bounds); targets } @@ -946,38 +983,32 @@ impl ElectionDataProvider> for Pallet } fn voters( - maybe_max_size: Option, + bounds: SnapshotBounds, ) -> data_provider::Result)>> { - Ok(match maybe_max_size { - Some(max_size) => Self::get_npos_voters_bounded(max_size), - None => { - log!( - warn, - "iterating over an unbounded number of npos voters, this might exhaust the \ - memory limits of the chain. Ensure proper limits are set via \ - `MaxNominatorsCount` or `ElectionProvider`" - ); - Self::get_npos_voters_unbounded() - }, + Ok(if bounds.is_unbounded() { + log!( + warn, + "iterating over an unbounded number of npos voters, this might exhaust the \ + memory limits of the chain. Ensure proper limits are set via \ + `MaxNominatorsCount` or `ElectionProvider`" + ); + Self::get_npos_voters_unbounded() + } else { + Self::get_npos_voters_bounded(bounds) }) } - fn targets(maybe_max_size: Option) -> data_provider::Result> { - // On any reasonable chain, the validator candidates should be small enough for this to not - // need to truncate. Nonetheless, if it happens, we prefer truncating for now rather than - // returning an error. In the future, a second instance of the `SortedListProvider` should - // be used to sort validators as well in a cheap way. - Ok(match maybe_max_size { - Some(max_size) => Self::get_npos_targets_bounded(max_size), - None => { - log!( - warn, - "iterating over an unbounded number of npos targets, this might exhaust the \ + fn targets(bounds: SnapshotBounds) -> data_provider::Result> { + Ok(if bounds.is_unbounded() { + log!( + warn, + "iterating over an unbounded number of npos targets, this might exhaust the \ memory limits of the chain. Ensure proper limits are set via \ `MaxValidatorsCount` or `ElectionProvider`" - ); - Self::get_npos_targets_unbounded() - }, + ); + Self::get_npos_targets_unbounded() + } else { + Self::get_npos_targets_bounded(bounds) }) } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 597166058f00a..054678cb48c7c 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3980,7 +3980,7 @@ fn on_finalize_weight_is_nonzero() { mod election_data_provider { use super::*; - use frame_election_provider_support::ElectionDataProvider; + use frame_election_provider_support::{ElectionDataProvider, SnapshotBounds}; #[test] fn voters_2sec_block() { @@ -4022,12 +4022,14 @@ mod election_data_provider { ExtBuilder::default().build_and_execute(|| { assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); assert_eq!( - >::voters(None) - .unwrap() - .iter() - .find(|x| x.0 == 101) - .unwrap() - .2, + >::voters( + SnapshotBounds::new_unbounded() + ) + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, vec![11, 21] ); @@ -4037,24 +4039,28 @@ mod election_data_provider { // 11 is gone. start_active_era(2); assert_eq!( - >::voters(None) - .unwrap() - .iter() - .find(|x| x.0 == 101) - .unwrap() - .2, + >::voters( + SnapshotBounds::new_unbounded() + ) + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, vec![21] ); // resubmit and it is back assert_ok!(Staking::nominate(Origin::signed(100), vec![11, 21])); assert_eq!( - >::voters(None) - .unwrap() - .iter() - .find(|x| x.0 == 101) - .unwrap() - .2, + >::voters( + SnapshotBounds::new_unbounded() + ) + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, vec![11, 21] ); }) @@ -4067,7 +4073,7 @@ mod election_data_provider { .build_and_execute(|| { let limit_for = |c| { Some( - Staking::voters(None) + Staking::voters(SnapshotBounds::new_unbounded()) .unwrap() .into_iter() .take(c) From 9844424cd49eb84d95d878070dffa7e0edbbb8e4 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 18 Nov 2021 10:49:50 +0000 Subject: [PATCH 3/7] fix a bunch of tests as well --- frame/staking/src/tests.rs | 69 ++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 054678cb48c7c..f0f6210056c60 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -4009,11 +4009,13 @@ mod election_data_provider { #[test] fn voters_include_self_vote() { ExtBuilder::default().nominate(false).build_and_execute(|| { - assert!(>::iter().map(|(x, _)| x).all(|v| Staking::voters(None) - .unwrap() - .into_iter() - .find(|(w, _, t)| { v == *w && t[0] == *w }) - .is_some())) + assert!(>::iter().map(|(x, _)| x).all(|v| Staking::voters( + SnapshotBounds::new_unbounded() + ) + .unwrap() + .into_iter() + .find(|(w, _, t)| { v == *w && t[0] == *w }) + .is_some())) }) } @@ -4072,13 +4074,13 @@ mod election_data_provider { .set_status(41, StakerStatus::Validator) .build_and_execute(|| { let limit_for = |c| { - Some( + SnapshotBounds::new_size( Staking::voters(SnapshotBounds::new_unbounded()) .unwrap() .into_iter() .take(c) .collect::>() - .encoded_size(), + .encoded_size() as u32, ) }; // sum of all nominators who'd be voters (1), plus the self-votes (4). @@ -4090,7 +4092,7 @@ mod election_data_provider { // unbounded: assert_eq!( - Staking::voters(None).unwrap(), + Staking::voters(SnapshotBounds::new_unbounded()).unwrap(), vec![ (101, 500, vec![11, 21]), // 8 + 8 + (8 * 2) + 1 = 33 (31, 500, vec![31]), // 8 + 8 + 8 + 1 = 25 @@ -4102,7 +4104,7 @@ mod election_data_provider { // if limits is less.. // let's check one of the manually for some mental practice - assert_eq!(limit_for(2).unwrap(), 33 + 25 + 1); + assert_eq!(limit_for(2).size_bound().unwrap(), 33 + 25 + 1); assert_eq!(Staking::voters(limit_for(2)).unwrap().len(), 2); // edge-case: we have enough size only for all validators, and none of the @@ -4114,7 +4116,14 @@ mod election_data_provider { assert_eq!(Staking::voters(limit_for(5)).unwrap().len(), 5); // if limit is more. - assert_eq!(Staking::voters(Some(limit_for(5).unwrap() * 2)).unwrap().len(), 5); + assert_eq!( + Staking::voters(SnapshotBounds::new_size( + (limit_for(5).size_bound().unwrap() * 2) as u32 + )) + .unwrap() + .len(), + 5 + ); }); } @@ -4124,13 +4133,13 @@ mod election_data_provider { .set_status(41, StakerStatus::Validator) .build_and_execute(|| { let limit_for = |c| { - Some( - Staking::targets(None) + SnapshotBounds::new_size( + Staking::targets(SnapshotBounds::new_unbounded()) .unwrap() .into_iter() .take(c) .collect::>() - .encoded_size(), + .encoded_size() as u32, ) }; @@ -4138,7 +4147,7 @@ mod election_data_provider { assert_eq!(>::iter().count() as u32, 4); // unbounded: - assert_eq!(Staking::targets(None).unwrap().len(), 4); + assert_eq!(Staking::targets(SnapshotBounds::new_unbounded()).unwrap().len(), 4); // if target limit is more.. assert_eq!(Staking::targets(limit_for(8)).unwrap().len(), 4); @@ -4164,14 +4173,16 @@ mod election_data_provider { assert_eq!(nominator_ids(), vec![101, 71, 61]); // we take 5 voters - let limit_5 = Staking::voters(None) - .unwrap() - .into_iter() - .take(5) - .collect::>() - .encoded_size(); + let limit_5 = SnapshotBounds::new_size( + Staking::voters(SnapshotBounds::new_unbounded()) + .unwrap() + .into_iter() + .take(5) + .collect::>() + .encoded_size() as u32, + ); assert_eq!( - Staking::voters(Some(limit_5)) + Staking::voters(limit_5) .unwrap() .iter() .map(|(stash, _, _)| stash) @@ -4191,14 +4202,16 @@ mod election_data_provider { add_slash(&21); // we take 4 voters - let limit_4 = Staking::voters(None) - .unwrap() - .into_iter() - .take(4) - .collect::>() - .encoded_size(); + let limit_4 = SnapshotBounds::new_size( + Staking::voters(SnapshotBounds::new_unbounded()) + .unwrap() + .into_iter() + .take(4) + .collect::>() + .encoded_size() as u32, + ); assert_eq!( - Staking::voters(Some(limit_4)) + Staking::voters(limit_4) .unwrap() .iter() .map(|(stash, _, _)| stash) From 7b4e7a8f3a3f34a302785f7dcf0085d69b3b1488 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Mon, 22 Nov 2021 17:01:09 +0100 Subject: [PATCH 4/7] it all works now --- bin/node/runtime/src/lib.rs | 29 +- frame/babe/src/mock.rs | 2 +- frame/bags-list/remote-tests/src/snapshot.rs | 3 +- .../src/benchmarking.rs | 4 +- .../election-provider-multi-phase/src/lib.rs | 12 +- frame/election-provider-support/src/lib.rs | 80 +- .../election-provider-support/src/onchain.rs | 2 +- frame/grandpa/src/mock.rs | 2 +- frame/offences/benchmarking/src/mock.rs | 2 +- frame/session/benchmarking/src/mock.rs | 2 +- frame/staking/src/benchmarking.rs | 6 +- frame/staking/src/pallet/impls.rs | 198 +++-- frame/staking/src/tests.rs | 689 +++++++++++++----- 13 files changed, 774 insertions(+), 257 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 064da1d59458f..f158d2064bce3 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -509,7 +509,7 @@ parameter_types! { pub OffchainRepeat: BlockNumber = 5; } -use frame_election_provider_support::onchain; +use frame_election_provider_support::{onchain, SnapshotBoundsBuilder}; impl onchain::Config for Runtime { type Accuracy = Perbill; type DataProvider = Staking; @@ -543,10 +543,11 @@ impl pallet_staking::Config for Runtime { // Note that the aforementioned does not scale to a very large number of nominators. type SortedListProvider = BagsList; // each nominator is allowed a fix number of nomination targets. - type NominationQuota = pallet_staking::FixedNominationQuota; + type NominationQuota = pallet_staking::FixedNominationQuota; type WeightInfo = pallet_staking::weights::SubstrateWeight; } +use frame_election_provider_support::SnapshotBounds; parameter_types! { // phase durations. 1/4 of the last session for each. pub const SignedPhase: u32 = EPOCH_DURATION_IN_BLOCKS / 4; @@ -572,9 +573,11 @@ parameter_types! { .max .get(DispatchClass::Normal); - // Our allocator can handle up to 32 MiB, we limit everything to 8 at most to be safe. - pub const VoterSnapshotSizePerBlock: u32 = 8 * 1024 * 1024; - pub const TargetSnapshotSize: u32 = 1 * 1024 * 1024; + /// maximum of 25k nominators, or 4MB. + pub VoterSnapshotBounds: SnapshotBounds = SnapshotBoundsBuilder::default().size(4 * 1024 * 1024).count(25_000).build(); + /// maximum of 1k validator candidates, with no size limit. + pub TargetSnapshotBounds: SnapshotBounds = SnapshotBounds::new_count(1000); + } sp_npos_elections::generate_solution_type!( @@ -656,8 +659,8 @@ impl pallet_election_provider_multi_phase::Config for Runtime { OffchainRandomBalancing, >; type ForceOrigin = EnsureRootOrHalfCouncil; - type VoterSnapshotSizePerBlock = VoterSnapshotSizePerBlock; - type TargetSnapshotSize = TargetSnapshotSize; + type VoterSnapshotBounds = VoterSnapshotBounds; + type TargetSnapshotBounds = TargetSnapshotBounds; type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; type BenchmarkingConfig = BenchmarkConfig; } @@ -1785,4 +1788,16 @@ mod tests { If the limit is too strong, maybe consider increase the limit to 300.", ); } + + #[test] + fn snapshot_details() { + let (_, _, max) = + pallet_staking::display_bounds_limits::(VoterSnapshotBounds::get()); + // example of an assertion that a runtime could have to ensure no mis-configuration happens. + assert!( + max <= 25_000, + "under any configuration, the maximum number of voters should be less than 25_000" + ); + let _ = pallet_staking::display_bounds_limits::(TargetSnapshotBounds::get()); + } } diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index e7ec692689032..5d83d8ad01680 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -196,7 +196,6 @@ impl onchain::Config for Test { } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; type RewardRemainder = (); type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; type Event = Event; @@ -216,6 +215,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type NominationQuota = pallet_staking::FixedNominationQuota<16>; type SortedListProvider = pallet_staking::UseNominatorsMap; } diff --git a/frame/bags-list/remote-tests/src/snapshot.rs b/frame/bags-list/remote-tests/src/snapshot.rs index 0e68a4495edfc..47555def1b07e 100644 --- a/frame/bags-list/remote-tests/src/snapshot.rs +++ b/frame/bags-list/remote-tests/src/snapshot.rs @@ -16,6 +16,7 @@ //! Test to execute the snapshot using the voter bag. +use frame_election_provider_support::SnapshotBounds; use frame_support::traits::PalletInfoAccess; use remote_externalities::{Builder, Mode, OnlineConfig}; use sp_runtime::{traits::Block as BlockT, DeserializeOwned}; @@ -57,7 +58,7 @@ pub async fn execute let voters = as ElectionDataProvider< Runtime::AccountId, Runtime::BlockNumber, - >>::voters(voter_limit) + >>::voters(SnapshotBounds::new_unbounded()) .unwrap(); let mut voters_nominator_only = voters diff --git a/frame/election-provider-multi-phase/src/benchmarking.rs b/frame/election-provider-multi-phase/src/benchmarking.rs index d9db6c3090994..9ca0bd345cad8 100644 --- a/frame/election-provider-multi-phase/src/benchmarking.rs +++ b/frame/election-provider-multi-phase/src/benchmarking.rs @@ -246,8 +246,8 @@ frame_benchmarking::benchmarks! { // we don't directly need the data-provider to be populated, but it is just easy to use it. set_up_data_provider::(v, t); - let targets = T::DataProvider::targets(None)?; - let voters = T::DataProvider::voters(None)?; + let targets = T::DataProvider::targets(SnapshotBounds::new_unbounded())?; + let voters = T::DataProvider::voters(SnapshotBounds::new_unbounded())?; let desired_targets = T::DataProvider::desired_targets()?; assert!(>::snapshot().is_none()); }: { diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index ad85e78ef2dfa..db2510e09c806 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -1315,12 +1315,12 @@ impl Pallet { T::DataProvider::desired_targets().map_err(ElectionError::DataProvider)?; // Defensive-only. - debug_assert!( - !voter_bound.exhausted(|| voters.encoded_size() as u32, || voters.len() as u32) - ); - debug_assert!( - !target_bound.exhausted(|| targets.encoded_size() as u32, || targets.len() as u32) - ); + debug_assert!(!voter_bound + .exhausts_size_count_non_zero(|| voters.encoded_size() as u32, || voters.len() as u32)); + debug_assert!(!target_bound.exhausts_size_count_non_zero( + || targets.encoded_size() as u32, + || targets.len() as u32 + )); Ok((targets, voters, desired_targets)) } diff --git a/frame/election-provider-support/src/lib.rs b/frame/election-provider-support/src/lib.rs index 47ea4a7f997bb..65d06ef82a889 100644 --- a/frame/election-provider-support/src/lib.rs +++ b/frame/election-provider-support/src/lib.rs @@ -161,6 +161,7 @@ #![cfg_attr(not(feature = "std"), no_std)] pub mod onchain; + use codec::{Decode, Encode}; use frame_support::{traits::Get, RuntimeDebug}; use sp_std::{fmt::Debug, prelude::*}; @@ -438,63 +439,118 @@ impl< /// The limits imposed on a snapshot, either voters or targets. #[derive(Clone, Copy, RuntimeDebug, scale_info::TypeInfo, Encode, Decode)] pub struct SnapshotBounds { + /// The bound on size, in bytes. `None` means unbounded. + size: Option, + /// The bound on count. `None` means unbounded. + count: Option, +} + +/// Utility builder for [`SnapshotBounds`]. +/// +/// The main purpose of this is to prevent mixing the order of similarly typed arguments (e.g. u32 +/// size and count). +#[derive(Default)] +pub struct SnapshotBoundsBuilder { size: Option, count: Option, } +impl SnapshotBoundsBuilder { + /// Set the given size. + pub fn size(mut self, size: u32) -> Self { + self.size = Some(size); + self + } + + /// Set the given count. + pub fn count(mut self, count: u32) -> Self { + self.count = Some(count); + self + } + + /// Build the [`SnapshotBounds`] instance. + pub fn build(self) -> SnapshotBounds { + SnapshotBounds { size: self.size, count: self.count } + } +} + impl SnapshotBounds { - pub fn new_size(size: u32) -> Self { + /// Create a new instance of self, with size limit. + pub const fn new_size(size: u32) -> Self { Self { size: Some(size), count: None } } - pub fn new_count(count: u32) -> Self { + /// Create a new instance of self, with count limit. + pub const fn new_count(count: u32) -> Self { Self { count: Some(count), size: None } } - pub fn new(count: u32, size: u32) -> Self { - Self { count: Some(count), size: Some(size) } - } - - pub fn new_unbounded() -> Self { + /// Create a new unbounded instance of self. + pub const fn new_unbounded() -> Self { Self { size: None, count: None } } + /// returns true if `given_size` exhausts `self.size`. pub fn size_exhausted(&self, given_size: impl FnOnce() -> u32) -> bool { self.size.map_or(false, |size| given_size() > size) } + /// returns true if `given_count` exhausts `self.count`. pub fn count_exhausted(&self, given_count: impl FnOnce() -> u32) -> bool { self.count.map_or(false, |count| given_count() > count) } - // TODO: don't like the signature, don't like the u32 a raw type that can get confused. Luckily - // it is not used all that much. - pub fn exhausted( + /// Returns true if `self` is exhausted by either of `given_size` and `given_count`. + /// + /// Note that this will return `false` against an empty contains (size = 1, count = 0). Calling + /// [`self.size_exhausted`] alone cannot handle this edge case, since no information of the + /// count is available. + /// + /// # Warning + /// + /// The function name is hinting at the correct order of `given_size` and `given_count`. Be + /// aware that they have the same type, and mixing them can be catastrophic. + pub fn exhausts_size_count_non_zero( &self, given_size: impl FnOnce() -> u32, given_count: impl FnOnce() -> u32, ) -> bool { - self.count_exhausted(given_count) || self.size_exhausted(given_size) + // take care of this pesky edge case: empty vector (size = 1, count = 0) should not exhaust + // anything. + let given_size = given_size(); + let given_count = given_count(); + if given_size == 1 || given_count == 0 { + return false + } + self.size_exhausted(|| given_size) || self.count_exhausted(|| given_count) } + /// Return the size bound, if one exists. pub fn size_bound(&self) -> Option { self.size.map(|b| b as usize) } + /// Return the count bound, if one exists. pub fn count_bound(&self) -> Option { self.count.map(|b| b as usize) } + /// Return `true` if self is fully unbounded. pub fn is_unbounded(&self) -> bool { self.size.is_none() && self.count.is_none() } + /// Return `true` if either of the bounds exists. pub fn is_bounded(&self) -> bool { !self.is_unbounded() } + /// Predict the `::with_capacity` of a collection that has `self` as bounds (size and count), + /// when each item is `item_size` bytes on average. + /// + /// Returns `None` if no capacity could be made (e.g. if `self` is unbounded). pub fn predict_capacity(&self, item_size: usize) -> Option { - match (self.size.map(|x| x as usize), self.count.map(|x| x as usize)) { + match (self.size_bound(), self.count_bound()) { (Some(max_size), Some(max_count)) => Some(max_count.min(max_size / item_size.max(1))), (Some(max_size), None) => Some(max_size / item_size.max(1)), (None, Some(max_count)) => Some(max_count), diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index 6b1fda77b9289..fa0fb465b2ea4 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -48,7 +48,7 @@ impl From for Error { /// how much weight was consumed. /// /// Finally, this implementation does not impose any limits on the number of voters and targets that -/// are provided. +/// are provided. Only use with a strictly bounded number of nominator/validator candidates. pub struct OnChainSequentialPhragmen(PhantomData); /// Configuration trait of [`OnChainSequentialPhragmen`]. diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index 49e4022a4aaed..fb1adfb6f20b3 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -198,7 +198,6 @@ impl onchain::Config for Test { } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; type RewardRemainder = (); type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; type Event = Event; @@ -218,6 +217,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type SortedListProvider = pallet_staking::UseNominatorsMap; + type NominationQuota = pallet_staking::FixedNominationQuota<16>; type WeightInfo = (); } diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index 3097f9b95be3f..8c30a0eb8904b 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -158,7 +158,6 @@ impl onchain::Config for Test { } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; type UnixTime = pallet_timestamp::Pallet; type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -178,6 +177,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type SortedListProvider = pallet_staking::UseNominatorsMap; + type NominationQuota = pallet_staking::FixedNominationQuota<16>; type WeightInfo = (); } diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index f534cc097e8a0..fa9534f1392c0 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -163,7 +163,6 @@ impl onchain::Config for Test { } impl pallet_staking::Config for Test { - const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; type UnixTime = pallet_timestamp::Pallet; type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; @@ -183,6 +182,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type SortedListProvider = pallet_staking::UseNominatorsMap; + type NominationQuota = pallet_staking::FixedNominationQuota<16>; type WeightInfo = (); } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 03e7015b23a30..899cb59dd101d 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -21,7 +21,7 @@ use super::*; use crate::Pallet as Staking; use testing_utils::*; -use frame_election_provider_support::SortedListProvider; +use frame_election_provider_support::{SnapshotBounds, SortedListProvider}; use frame_support::{ dispatch::UnfilteredDispatchable, pallet_prelude::*, @@ -848,7 +848,7 @@ benchmarks! { let num_voters = (v + n) as usize; }: { - let voters = >::get_npos_voters_bounded(Bounded::max_value()); + let voters = >::get_npos_voters_bounded(SnapshotBounds::new_unbounded()); assert_eq!(voters.len(), num_voters); } @@ -872,7 +872,7 @@ benchmarks! { v, 0, 0, false, None )?; }: { - let targets = >::get_npos_targets_bounded(Bounded::max_value()); + let targets = >::get_npos_targets_bounded(SnapshotBounds::new_unbounded()); assert_eq!(targets.len() as u32, v); } diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 40734a186e6d0..7e00b0120e3a2 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -651,6 +651,11 @@ impl Pallet { /// /// This function is self-weighing as [`DispatchClass::Mandatory`]. /// + /// ### Slashing + /// + /// All nominations that have been submitted before the last non-zero slash of the validator are + /// auto-chilled. + /// /// # Warning /// /// This is the unbounded variant. Being called might cause a large number of storage reads. Use @@ -664,12 +669,12 @@ impl Pallet { if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = Self::nominators(nominator.clone()) { - targets.retain(|stash| { + targets.retain(|validator_stash| { slashing_spans - .get(stash) + .get(validator_stash) .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) }); - if !targets.len().is_zero() { + if !targets.is_empty() { let weight = weight_of(&nominator); return Some((nominator, weight, targets)) } @@ -697,15 +702,16 @@ impl Pallet { // NOTE: we chain the one we expect to have the smaller size (`validators_votes`) to the // larger one, to minimize copying. Ideally we would collect only once, but sadly then we - // wouldn't have access to a cheap `.len()`, which we need for weighing. TODO: maybe - // `.count()` is still cheaper than the copying we do. + // wouldn't have access to a cheap `.len()`, which we need for weighing. Only other option + // would have been using the counters, but since we entirely avoid reading them, we better + // stick to that. nominator_votes.extend(validator_votes); nominator_votes } /// Get all of the voters that are eligible for the next npos election. /// - /// `bounds` imposes a cap on the count and byte-size of the entire voters returned. + /// `bounds` imposes a cap on the count and byte-size of the entire vector returned. /// /// As of now, first all the validator are included in no particular order, then remainder is /// taken from the nominators, as returned by [`Config::SortedListProvider`]. @@ -731,12 +737,18 @@ impl Pallet { Vec::new() }; + // we create two closures to make us agnostic of the type of `bounds` that we are dealing + // with. This still not the optimum. The most performant option would have been having a + // dedicated function for each variant. For example, in the current code, if `bounds` + // count-bounded, the static size tracker is allocated for now reason. Nonetheless, it is + // not actually tracking anything if it is not needed. + // register a voter with `votes` with regards to bounds. let add_voter = |tracker_ref: &mut StaticSizeTracker, votes: usize| { if let Some(_) = bounds.size_bound() { tracker_ref.register_voter(votes) } else if let Some(_) = bounds.count_bound() { - // nothing needed, voters.len() is itself a representation. + // nothing needed, voters.len() is itself a representation of the count. } }; @@ -747,11 +759,11 @@ impl Pallet { bounds.count_bound(), ) { (Some(max_size), Some(max_count)) => - tracker_ref.final_byte_size_of(voters_ref.len() + 1) > max_size || - voters_ref.len() + 1 > max_count, + tracker_ref.final_byte_size_of(voters_ref.len().saturating_add(1)) > max_size || + voters_ref.len().saturating_add(1) > max_count, (Some(max_size), None) => - tracker_ref.final_byte_size_of(voters_ref.len() + 1) > max_size, - (None, Some(max_count)) => voters_ref.len() + 1 > max_count, + tracker_ref.final_byte_size_of(voters_ref.len().saturating_add(1)) > max_size, + (None, Some(max_count)) => voters_ref.len().saturating_add(1) > max_count, (None, None) => false, }; @@ -774,42 +786,58 @@ impl Pallet { } let validators_taken = voters.len(); - // cache a few items. - let slashing_spans = >::iter().collect::>(); - let weight_of = Self::weight_of_fn(); + // only bother with reading the slashing spans et.al. if we are not exhausted. + let slashing_spans_read = if !next_will_exhaust(&tracker, &voters) { + let slashing_spans = >::iter().collect::>(); + let weight_of = Self::weight_of_fn(); - for nominator in T::SortedListProvider::iter() { - if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = - >::get(&nominator) - { - // IMPORTANT: we track the size and potentially break out right here. This ensures - // that votes that are invalid will also affect the snapshot bounds. Chain operators - // should ensure `update_slashed_nominator` is used to eliminate the need for this. - add_voter(&mut tracker, targets.len()); - if next_will_exhaust(&tracker, &voters) { - break - } + for nominator in T::SortedListProvider::iter() { + if let Some(Nominations { submitted_in, mut targets, suppressed: _ }) = + >::get(&nominator) + { + // IMPORTANT: we track the size and potentially break out right here. This + // ensures that votes that are invalid will also affect the snapshot bounds. + // Chain operators should ensure `update_slashed_nominator` is used to eliminate + // the need for this. + add_voter(&mut tracker, targets.len()); + if next_will_exhaust(&tracker, &voters) { + break + } - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); - if !targets.len().is_zero() { - voters.push((nominator.clone(), weight_of(&nominator), targets)); + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + if !targets.len().is_zero() { + voters.push((nominator.clone(), weight_of(&nominator), targets)); + } + } else { + log!(error, "DEFENSIVE: invalid item in `SortedListProvider`: {:?}", nominator) } - } else { - log!(error, "DEFENSIVE: invalid item in `SortedListProvider`: {:?}", nominator) } - } + slashing_spans.len() as u32 + } else { + Zero::zero() + }; let nominators_taken = voters.len().saturating_sub(validators_taken); Self::register_weight(T::WeightInfo::get_npos_voters_bounded( validators_taken as u32, nominators_taken as u32, - slashing_spans.len() as u32, + slashing_spans_read, )); - debug_assert!(!bounds.exhausted(|| voters.encoded_size() as u32, || voters.len() as u32)); + + debug_assert!( + !bounds.exhausts_size_count_non_zero( + || voters.encoded_size() as u32, + || voters.len() as u32 + ), + "{} voters, size {}, exhausted {:?}", + voters.len(), + voters.encoded_size(), + bounds, + ); log!( info, @@ -878,7 +906,10 @@ impl Pallet { } Self::register_weight(T::WeightInfo::get_npos_targets_bounded(targets.len() as u32)); - debug_assert!(!bounds.exhausted(|| targets.encoded_size() as u32, || targets.len() as u32)); + debug_assert!(!bounds.exhausts_size_count_non_zero( + || targets.encoded_size() as u32, + || targets.len() as u32 + )); log!(info, "generated {} npos targets, with bounds limit {:?}", targets.len(), bounds); targets @@ -1459,7 +1490,7 @@ impl SortedListProvider for UseNominatorsMap { /// Whilst doing this, [`size`] will track the entire size of the `Vec`, except for the /// length prefix of the outer `Vec`. To get the final size at any point, use /// [`final_byte_size_of`]. -struct StaticSizeTracker { +pub(crate) struct StaticSizeTracker { size: usize, _marker: sp_std::marker::PhantomData, } @@ -1471,7 +1502,7 @@ impl StaticSizeTracker { /// The length prefix of a vector with the given length. #[inline] - fn length_prefix(length: usize) -> usize { + pub(crate) fn length_prefix(length: usize) -> usize { // TODO: scale codec could and should expose a public function for this that I can reuse. match length { 0..=63 => 1, @@ -1485,12 +1516,12 @@ impl StaticSizeTracker { } /// Register a voter in `self` who has casted `votes`. - fn register_voter(&mut self, votes: usize) { + pub(crate) fn register_voter(&mut self, votes: usize) { self.size = self.size.saturating_add(Self::voter_size(votes)) } /// The byte size of a voter who casted `votes`. - fn voter_size(votes: usize) -> usize { + pub(crate) fn voter_size(votes: usize) -> usize { Self::length_prefix(votes) // and each element .saturating_add(votes * sp_std::mem::size_of::()) @@ -1501,7 +1532,7 @@ impl StaticSizeTracker { } // Final size: size of all internal elements, plus the length prefix. - fn final_byte_size_of(&self, length: usize) -> usize { + pub(crate) fn final_byte_size_of(&self, length: usize) -> usize { self.size + Self::length_prefix(length) } } @@ -1550,3 +1581,84 @@ mod static_tracker { assert_eq!(voters.encoded_size(), tracker.final_byte_size_of(voters.len())); } } + +/// A helper function that does nothing other than return some information about the range at which +/// the given `bounds` works. +/// +/// Will print and return as a tuple as `(low, mid, high)`, where: +/// +/// - `low` is the minimum number of voters that `bounds` supports. This will be realized when all +/// voters use [`T::NominationQuota::ABSOLUTE_MAXIMUM`] votes. +/// - `how` is the maximum number of voters that `bounds` supports. This will be realized when all +/// voters use `1` votes. +/// - `mid` is the the average of the above two. This will be realized when all voters use +/// `[`T::NominationQuota::ABSOLUTE_MAXIMUM`] / 2` votes. +#[cfg(feature = "std")] +pub fn display_bounds_limits(bounds: SnapshotBounds) -> (usize, usize, usize) { + match (bounds.size_bound(), bounds.count_bound()) { + (None, None) => { + println!("{:?} is unbounded", bounds); + (Bounded::max_value(), Bounded::max_value(), Bounded::max_value()) + }, + (None, Some(count)) => { + println!("{:?} can have exactly maximum {} voters", bounds, count); + (count, count, count) + }, + (Some(size), Some(count)) => { + // maximum number of voters, it means that they all voted 1. + let max_voters = { + let voter_size = StaticSizeTracker::::voter_size(1); + // assuming that the length prefix is 4 bytes. + (size.saturating_sub(4) / voter_size).min(count) + }; + // minimum number of voters, it means that they all voted maximum. + let min_voters = { + let voter_size = StaticSizeTracker::::voter_size( + T::NominationQuota::ABSOLUTE_MAXIMUM as usize, + ); + (size.saturating_sub(4) / voter_size).min(count) + }; + // average of the above two. + let average_voters = { + let voter_size = StaticSizeTracker::::voter_size( + T::NominationQuota::ABSOLUTE_MAXIMUM as usize / 2, + ); + // assuming that the length prefix is 4 bytes. + (size.saturating_sub(4) / voter_size).min(count) + }; + println!( + "{:?} will be in [low, mid, high]: [{}, {}, {}]", + bounds, min_voters, average_voters, max_voters + ); + (min_voters, average_voters, max_voters) + }, + (Some(size), None) => { + // maximum number of voters, it means that they all voted 1. + let max_voters = { + let voter_size = StaticSizeTracker::::voter_size(1); + // assuming that the length prefix is 4 bytes. + size.saturating_sub(4) / voter_size + }; + // minimum number of voters, it means that they all voted maximum. + let min_voters = { + let voter_size = StaticSizeTracker::::voter_size( + T::NominationQuota::ABSOLUTE_MAXIMUM as usize, + ); + size.saturating_sub(4) / voter_size + }; + // average of the above two. + let average_voters = { + let voter_size = StaticSizeTracker::::voter_size( + T::NominationQuota::ABSOLUTE_MAXIMUM as usize / 2, + ); + // assuming that the length prefix is 4 bytes. + size.saturating_sub(4) / voter_size + }; + println!( + "{:?} will be in [low, mid, high]: [{}, {}, {}]", + bounds, min_voters, average_voters, max_voters + ); + (min_voters, average_voters, max_voters) + }, + } +} diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 1495431dcffd2..61c2a58d43027 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3954,7 +3954,72 @@ fn on_finalize_weight_is_nonzero() { mod election_data_provider { use super::*; - use frame_election_provider_support::{ElectionDataProvider, SnapshotBounds}; + use frame_election_provider_support::{ + ElectionDataProvider, SnapshotBounds, SnapshotBoundsBuilder, + }; + + fn all_voters_count() -> u32 { + CounterForValidators::::get() + CounterForNominators::::get() + } + + fn limit_for_voters(maybe_size: Option, maybe_count: Option) -> SnapshotBounds { + let mut builder = SnapshotBoundsBuilder::default(); + if let Some(size) = maybe_size { + let all_voters = Staking::voters(SnapshotBounds::new_unbounded()).unwrap(); + let size_bound = if size <= all_voters.len() { + all_voters.into_iter().take(size).collect::>().encoded_size() as u32 + } else { + let excess = size - all_voters.len(); + let base = all_voters.encoded_size(); + let per_item = base / all_voters.len(); + (base + per_item * excess) as u32 + }; + builder = builder.size(size_bound) + } + + if let Some(count) = maybe_count { + builder = builder.count(count as u32) + } + + builder.build() + } + + fn limit_for_targets(maybe_size: Option, maybe_count: Option) -> SnapshotBounds { + let mut builder = SnapshotBoundsBuilder::default(); + if let Some(size) = maybe_size { + let all_targets = Staking::targets(SnapshotBounds::new_unbounded()).unwrap(); + let size_bound = if size <= all_targets.len() { + dbg!(all_targets.iter().take(size).collect::>().encoded_size() as u32) + } else { + let excess = size - all_targets.len(); + let base = all_targets.encoded_size(); + let per_item = base / all_targets.len(); + (base + per_item * excess) as u32 + }; + builder = builder.size(size_bound) + } + if let Some(count) = maybe_count { + builder = builder.count(count as u32); + } + + builder.build() + } + + fn count_limit_for_voters(count: usize) -> SnapshotBounds { + limit_for_voters(None, Some(count)) + } + + fn size_limit_for_voters(size: usize) -> SnapshotBounds { + limit_for_voters(Some(size), None) + } + + fn count_limit_for_targets(count: usize) -> SnapshotBounds { + limit_for_targets(None, Some(count)) + } + + fn size_limit_for_targets(size: usize) -> SnapshotBounds { + limit_for_targets(Some(size), None) + } #[test] fn voters_2sec_block() { @@ -3981,156 +4046,477 @@ mod election_data_provider { } #[test] - fn voters_include_self_vote() { - ExtBuilder::default().nominate(false).build_and_execute(|| { - assert!(>::iter().map(|(x, _)| x).all(|v| Staking::voters( - SnapshotBounds::new_unbounded() - ) - .unwrap() - .into_iter() - .find(|(w, _, t)| { v == *w && t[0] == *w }) - .is_some())) - }) - } + fn estimate_next_election_works() { + ExtBuilder::default().session_per_era(5).period(5).build_and_execute(|| { + // first session is always length 0. + for b in 1..20 { + run_to_block(b); + assert_eq!(Staking::next_election_prediction(System::block_number()), 20); + } - #[test] - fn voters_exclude_slashed() { - ExtBuilder::default().build_and_execute(|| { - assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); - assert_eq!( - >::voters( - SnapshotBounds::new_unbounded() - ) - .unwrap() - .iter() - .find(|x| x.0 == 101) - .unwrap() - .2, - vec![11, 21] - ); + // election + run_to_block(20); + assert_eq!(Staking::next_election_prediction(System::block_number()), 45); + assert_eq!(staking_events().len(), 1); + assert_eq!(*staking_events().last().unwrap(), Event::StakersElected); - start_active_era(1); - add_slash(&11); + for b in 21..45 { + run_to_block(b); + assert_eq!(Staking::next_election_prediction(System::block_number()), 45); + } - // 11 is gone. - start_active_era(2); - assert_eq!( - >::voters( - SnapshotBounds::new_unbounded() - ) - .unwrap() - .iter() - .find(|x| x.0 == 101) - .unwrap() - .2, - vec![21] - ); + // election + run_to_block(45); + assert_eq!(Staking::next_election_prediction(System::block_number()), 70); + assert_eq!(staking_events().len(), 3); + assert_eq!(*staking_events().last().unwrap(), Event::StakersElected); - // resubmit and it is back - assert_ok!(Staking::nominate(Origin::signed(100), vec![11, 21])); - assert_eq!( - >::voters( - SnapshotBounds::new_unbounded() - ) - .unwrap() - .iter() - .find(|x| x.0 == 101) - .unwrap() - .2, - vec![11, 21] - ); + Staking::force_no_eras(Origin::root()).unwrap(); + assert_eq!(Staking::next_election_prediction(System::block_number()), u64::MAX); + + Staking::force_new_era_always(Origin::root()).unwrap(); + assert_eq!(Staking::next_election_prediction(System::block_number()), 45 + 5); + + Staking::force_new_era(Origin::root()).unwrap(); + assert_eq!(Staking::next_election_prediction(System::block_number()), 45 + 5); + + // Do a fail election + MinimumValidatorCount::::put(1000); + run_to_block(50); + // Election: failed, next session is a new election + assert_eq!(Staking::next_election_prediction(System::block_number()), 50 + 5); + // The new era is still forced until a new era is planned. + assert_eq!(ForceEra::::get(), Forcing::ForceNew); + + MinimumValidatorCount::::put(2); + run_to_block(55); + assert_eq!(Staking::next_election_prediction(System::block_number()), 55 + 25); + assert_eq!(staking_events().len(), 6); + assert_eq!(*staking_events().last().unwrap(), Event::StakersElected); + // The new era has been planned, forcing is changed from `ForceNew` to `NotForcing`. + assert_eq!(ForceEra::::get(), Forcing::NotForcing); }) } - #[test] - fn get_npos_voters_works() { - ExtBuilder::default() - .set_status(41, StakerStatus::Validator) - .build_and_execute(|| { - let limit_for = |c| { - SnapshotBounds::new_size( - Staking::voters(SnapshotBounds::new_unbounded()) - .unwrap() - .into_iter() - .take(c) - .collect::>() - .encoded_size() as u32, + mod bounded { + use super::*; + + #[test] + fn voters_include_self_vote() { + ExtBuilder::default().nominate(false).build_and_execute(|| { + assert!(>::iter().map(|(x, _)| x).all(|v| Staking::voters( + SnapshotBounds::new_count(all_voters_count()) + ) + .unwrap() + .into_iter() + .find(|(w, _, t)| { v == *w && t[0] == *w }) + .is_some())) + }) + } + + #[test] + fn voters_exclude_slashed() { + ExtBuilder::default().build_and_execute(|| { + assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); + assert_eq!( + >::voters( + SnapshotBounds::new_count(all_voters_count()) ) - }; - // sum of all nominators who'd be voters (1), plus the self-votes (4). + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![11, 21] + ); + + start_active_era(1); + add_slash(&11); + + // 11 is gone. + start_active_era(2); assert_eq!( - ::SortedListProvider::count() + - >::iter().count() as u32, - 5 + >::voters( + SnapshotBounds::new_count(all_voters_count()) + ) + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![21] ); - // unbounded: + // resubmit and it is back + assert_ok!(Staking::nominate(Origin::signed(100), vec![11, 21])); assert_eq!( - Staking::voters(SnapshotBounds::new_unbounded()).unwrap(), - vec![ - (101, 500, vec![11, 21]), // 8 + 8 + (8 * 2) + 1 = 33 - (31, 500, vec![31]), // 8 + 8 + 8 + 1 = 25 - (41, 1000, vec![41]), - (21, 1000, vec![21]), - (11, 1000, vec![11]), - ] + >::voters( + SnapshotBounds::new_count(all_voters_count()) + ) + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![11, 21] ); + }) + } + + #[test] + fn get_npos_voters_works_size_limit() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // unbounded: + assert_eq!( + Staking::voters(SnapshotBounds::new_unbounded()).unwrap(), + vec![ + (101, 500, vec![11, 21]), // 8 + 8 + (8 * 2) + 1 = 33 + (31, 500, vec![31]), // 8 + 8 + 8 + 1 = 25 + (41, 1000, vec![41]), + (21, 1000, vec![21]), + (11, 1000, vec![11]), + ] + ); + + // if limits is less.. + assert_eq!(Staking::voters(SnapshotBounds::new_size(0)).unwrap().len(), 0); + + // let's check one of the manually for some mental practice + assert_eq!(size_limit_for_voters(2).size_bound().unwrap(), 33 + 25 + 1); + assert_eq!(Staking::voters(size_limit_for_voters(2)).unwrap().len(), 2); + + // edge-case: we have enough size only for all validators, and none of the + // nominators. + let limit_validators = + size_limit_for_voters(>::iter().count()); + let voters = Staking::voters(limit_validators).unwrap(); + assert_eq!(voters.len(), 4); + assert!(Validators::::iter() + .all(|(v, _)| voters.iter().filter(|x| x.0 == v).count() == 1)); + + // if limit is equal.. + assert_eq!(Staking::voters(size_limit_for_voters(5)).unwrap().len(), 5); + + // if limit is more. + assert_eq!(Staking::voters(size_limit_for_voters(10)).unwrap().len(), 5); + }); + } + + #[test] + fn get_npos_voters_works_count_limit() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // sum of all nominators who'd be voters (1), plus the self-votes (4). + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 5 + ); + + // unbounded: + assert_eq!(Staking::voters(SnapshotBounds::new_unbounded()).unwrap().len(), 5); + + // if less.. + assert_eq!(Staking::voters(count_limit_for_voters(0)).unwrap().len(), 0); + assert_eq!(Staking::voters(count_limit_for_voters(2)).unwrap().len(), 2); + + // only validators edge case.. + let voters = Staking::voters(count_limit_for_voters(4)).unwrap(); + assert_eq!(voters.len(), 4); + assert!(Validators::::iter() + .all(|(v, _)| voters.iter().filter(|x| x.0 == v).count() == 1)); + + // equal.. + assert_eq!(Staking::voters(count_limit_for_voters(5)).unwrap().len(), 5); + + // and finally more. + assert_eq!(Staking::voters(count_limit_for_voters(7)).unwrap().len(), 5); + }); + } - // if limits is less.. - // let's check one of the manually for some mental practice - assert_eq!(limit_for(2).size_bound().unwrap(), 33 + 25 + 1); - assert_eq!(Staking::voters(limit_for(2)).unwrap().len(), 2); + #[test] + fn get_npos_voters_works_hybrid_limit() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // sum of all nominators who'd be voters (1), plus the self-votes (4). + assert_eq!( + ::SortedListProvider::count() + + >::iter().count() as u32, + 5 + ); + + // if less.. + assert_eq!( + Staking::voters(limit_for_voters(Some(0), Some(0))).unwrap().len(), + 0 + ); + assert_eq!( + Staking::voters(limit_for_voters(Some(0), Some(2))).unwrap().len(), + 0 + ); + assert_eq!( + Staking::voters(limit_for_voters(Some(2), Some(0))).unwrap().len(), + 0 + ); + + assert_eq!( + Staking::voters(limit_for_voters(Some(2), Some(1))).unwrap().len(), + 1 + ); + assert_eq!( + Staking::voters(limit_for_voters(Some(1), Some(1))).unwrap().len(), + 1 + ); + assert_eq!( + Staking::voters(limit_for_voters(Some(1), Some(2))).unwrap().len(), + 1 + ); + + assert_eq!( + Staking::voters(limit_for_voters(Some(2), Some(2))).unwrap().len(), + 2 + ); + + // only validators edge case.. + let v1 = Staking::voters(limit_for_voters(Some(4), Some(4))).unwrap(); + let v2 = Staking::voters(limit_for_voters(Some(4), Some(5))).unwrap(); + let v3 = Staking::voters(limit_for_voters(Some(5), Some(4))).unwrap(); + assert_eq!(v1, v2); + assert_eq!(v1, v3); + assert_eq!(v1.len(), 4); + assert!(Validators::::iter() + .all(|(v, _)| v1.iter().filter(|x| x.0 == v).count() == 1)); + + // equal.. + assert_eq!( + Staking::voters(limit_for_voters(Some(5), Some(5))).unwrap().len(), + 5 + ); + assert_eq!( + Staking::voters(limit_for_voters(Some(5), Some(6))).unwrap().len(), + 5 + ); + assert_eq!( + Staking::voters(limit_for_voters(Some(6), Some(5))).unwrap().len(), + 5 + ); + + // and finally more. + assert_eq!( + Staking::voters(limit_for_voters(Some(7), Some(7))).unwrap().len(), + 5 + ); + assert_eq!( + Staking::voters(limit_for_voters(Some(10), Some(7))).unwrap().len(), + 5 + ); + }); + } - // edge-case: we have enough size only for all validators, and none of the - // nominators. - let limit_validators = limit_for(>::iter().count()); - assert_eq!(Staking::voters(limit_validators).unwrap().len(), 4); + #[test] + fn get_npos_targets_works_size_limit() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // all targets: + assert_eq!(>::iter().count() as u32, 4); + + // if target limit is less.. + assert_eq!(Staking::targets(size_limit_for_targets(0)).unwrap().len(), 0); + assert_eq!(Staking::targets(size_limit_for_targets(1)).unwrap().len(), 1); + assert_eq!(Staking::targets(size_limit_for_targets(3)).unwrap().len(), 3); + + // if target limit is more.. + assert_eq!(Staking::targets(size_limit_for_targets(4)).unwrap().len(), 4); + assert_eq!(Staking::targets(size_limit_for_targets(8)).unwrap().len(), 4); + }) + } - // if limit is equal.. - assert_eq!(Staking::voters(limit_for(5)).unwrap().len(), 5); + #[test] + fn get_npos_targets_works_count_limit() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // all targets: + assert_eq!(>::iter().count() as u32, 4); + + // if target limit is less.. + assert_eq!(Staking::targets(count_limit_for_targets(0)).unwrap().len(), 0); + assert_eq!(Staking::targets(count_limit_for_targets(1)).unwrap().len(), 1); + assert_eq!(Staking::targets(count_limit_for_targets(3)).unwrap().len(), 3); + + // if target limit is more.. + assert_eq!(Staking::targets(count_limit_for_targets(4)).unwrap().len(), 4); + assert_eq!(Staking::targets(count_limit_for_targets(8)).unwrap().len(), 4); + }) + } + + #[test] + fn get_npos_targets_works_hybrid_limit() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // sum of all validators/targets. + assert_eq!(CounterForValidators::::get(), 4); + + // if less.. + assert_eq!( + Staking::targets(limit_for_targets(Some(0), Some(0))).unwrap().len(), + 0 + ); + assert_eq!( + Staking::targets(limit_for_targets(Some(0), Some(2))).unwrap().len(), + 0 + ); + assert_eq!( + Staking::targets(limit_for_targets(Some(2), Some(0))).unwrap().len(), + 0 + ); + + assert_eq!( + Staking::targets(dbg!(limit_for_targets(Some(2), Some(1)))).unwrap().len(), + 1 + ); + assert_eq!( + Staking::targets(limit_for_targets(Some(1), Some(1))).unwrap().len(), + 1 + ); + assert_eq!( + Staking::targets(limit_for_targets(Some(1), Some(2))).unwrap().len(), + 1 + ); + assert_eq!( + Staking::targets(limit_for_targets(Some(2), Some(2))).unwrap().len(), + 2 + ); + + // almost equal.. + assert_eq!( + Staking::targets(limit_for_targets(Some(3), Some(4))).unwrap().len(), + 3 + ); + // just equal.. + assert_eq!( + Staking::targets(limit_for_targets(Some(4), Some(4))).unwrap().len(), + 4 + ); + + // if more + assert_eq!( + Staking::targets(limit_for_targets(Some(5), Some(6))).unwrap().len(), + 4 + ); + assert_eq!( + Staking::targets(limit_for_targets(Some(6), Some(5))).unwrap().len(), + 4 + ); + }); + } + } + + mod unbounded { + use super::*; + + #[test] + fn voters_include_self_vote() { + ExtBuilder::default().nominate(false).build_and_execute(|| { + assert!(>::iter().map(|(x, _)| x).all(|v| Staking::voters( + SnapshotBounds::new_count(all_voters_count()) + ) + .unwrap() + .into_iter() + .find(|(w, _, t)| { v == *w && t[0] == *w }) + .is_some())) + }) + } - // if limit is more. + #[test] + fn voters_exclude_slashed() { + ExtBuilder::default().build_and_execute(|| { + assert_eq!(Staking::nominators(101).unwrap().targets, vec![11, 21]); assert_eq!( - Staking::voters(SnapshotBounds::new_size( - (limit_for(5).size_bound().unwrap() * 2) as u32 - )) + >::voters( + SnapshotBounds::new_unbounded() + ) .unwrap() - .len(), - 5 + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![11, 21] ); - }); - } - #[test] - fn respects_targets_snapshot_len_limits() { - ExtBuilder::default() - .set_status(41, StakerStatus::Validator) - .build_and_execute(|| { - let limit_for = |c| { - SnapshotBounds::new_size( - Staking::targets(SnapshotBounds::new_unbounded()) - .unwrap() - .into_iter() - .take(c) - .collect::>() - .encoded_size() as u32, + start_active_era(1); + add_slash(&11); + + // 11 is gone. + start_active_era(2); + assert_eq!( + >::voters( + SnapshotBounds::new_unbounded() ) - }; + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![21] + ); - // all targets: - assert_eq!(>::iter().count() as u32, 4); + // resubmit and it is back + assert_ok!(Staking::nominate(Origin::signed(100), vec![11, 21])); + assert_eq!( + >::voters( + SnapshotBounds::new_unbounded() + ) + .unwrap() + .iter() + .find(|x| x.0 == 101) + .unwrap() + .2, + vec![11, 21] + ); + }) + } - // unbounded: - assert_eq!(Staking::targets(SnapshotBounds::new_unbounded()).unwrap().len(), 4); + #[test] + fn get_npos_voters_works() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + assert_eq!( + CounterForNominators::::get() + CounterForValidators::::get(), + 5 + ); + assert_eq!( + Staking::voters(SnapshotBounds::new_unbounded()).unwrap(), + vec![ + (101, 500, vec![11, 21]), + (31, 500, vec![31]), + (41, 1000, vec![41]), + (21, 1000, vec![21]), + (11, 1000, vec![11]), + ] + ); + }) + } - // if target limit is more.. - assert_eq!(Staking::targets(limit_for(8)).unwrap().len(), 4); - assert_eq!(Staking::targets(limit_for(4)).unwrap().len(), 4); + #[test] + fn get_npos_targets_works() { + ExtBuilder::default() + .set_status(41, StakerStatus::Validator) + .build_and_execute(|| { + // all targets: + assert_eq!(>::iter().count() as u32, 4); - // if target limit is less.. - assert_eq!(Staking::targets(limit_for(3)).unwrap().len(), 3); - assert_eq!(Staking::targets(limit_for(1)).unwrap().len(), 1); - }) + // unbounded: + assert_eq!(Staking::targets(SnapshotBounds::new_unbounded()).unwrap().len(), 4); + }) + } } // Even if some of the higher staked nominators are slashed, we don't get up to max len voters @@ -4200,59 +4586,6 @@ mod election_data_provider { ); }); } - - #[test] - fn estimate_next_election_works() { - ExtBuilder::default().session_per_era(5).period(5).build_and_execute(|| { - // first session is always length 0. - for b in 1..20 { - run_to_block(b); - assert_eq!(Staking::next_election_prediction(System::block_number()), 20); - } - - // election - run_to_block(20); - assert_eq!(Staking::next_election_prediction(System::block_number()), 45); - assert_eq!(staking_events().len(), 1); - assert_eq!(*staking_events().last().unwrap(), Event::StakersElected); - - for b in 21..45 { - run_to_block(b); - assert_eq!(Staking::next_election_prediction(System::block_number()), 45); - } - - // election - run_to_block(45); - assert_eq!(Staking::next_election_prediction(System::block_number()), 70); - assert_eq!(staking_events().len(), 3); - assert_eq!(*staking_events().last().unwrap(), Event::StakersElected); - - Staking::force_no_eras(Origin::root()).unwrap(); - assert_eq!(Staking::next_election_prediction(System::block_number()), u64::MAX); - - Staking::force_new_era_always(Origin::root()).unwrap(); - assert_eq!(Staking::next_election_prediction(System::block_number()), 45 + 5); - - Staking::force_new_era(Origin::root()).unwrap(); - assert_eq!(Staking::next_election_prediction(System::block_number()), 45 + 5); - - // Do a fail election - MinimumValidatorCount::::put(1000); - run_to_block(50); - // Election: failed, next session is a new election - assert_eq!(Staking::next_election_prediction(System::block_number()), 50 + 5); - // The new era is still forced until a new era is planned. - assert_eq!(ForceEra::::get(), Forcing::ForceNew); - - MinimumValidatorCount::::put(2); - run_to_block(55); - assert_eq!(Staking::next_election_prediction(System::block_number()), 55 + 25); - assert_eq!(staking_events().len(), 6); - assert_eq!(*staking_events().last().unwrap(), Event::StakersElected); - // The new era has been planned, forcing is changed from `ForceNew` to `NotForcing`. - assert_eq!(ForceEra::::get(), Forcing::NotForcing); - }) - } } #[test] From 2f629a9ccdcc751f4ec52a04c9d9451470271d38 Mon Sep 17 00:00:00 2001 From: Kian Paimani Date: Wed, 24 Nov 2021 09:45:03 +0100 Subject: [PATCH 5/7] A test commit from the new macbook --- Cargo.lock | 148 +++++++----- Cargo.toml | 1 + frame/election-playground/Cargo.toml | 42 ++++ frame/election-playground/src/lib.rs | 65 ++++++ frame/election-playground/src/mock.rs | 315 ++++++++++++++++++++++++++ frame/staking/src/pallet/impls.rs | 16 +- 6 files changed, 514 insertions(+), 73 deletions(-) create mode 100644 frame/election-playground/Cargo.toml create mode 100644 frame/election-playground/src/lib.rs create mode 100644 frame/election-playground/src/mock.rs diff --git a/Cargo.lock b/Cargo.lock index a45e0accd49b7..7c57ff25b592a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -480,7 +480,7 @@ dependencies = [ "futures 0.3.16", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-client-api", "sc-keystore", "sc-network", @@ -1857,7 +1857,7 @@ dependencies = [ "log 0.4.14", "num-traits", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.8.4", "scale-info", ] @@ -3087,7 +3087,7 @@ dependencies = [ "jsonrpc-server-utils", "log 0.4.14", "net2", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "unicase 2.6.0", ] @@ -3102,7 +3102,7 @@ dependencies = [ "jsonrpc-server-utils", "log 0.4.14", "parity-tokio-ipc", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "tower-service", ] @@ -3116,7 +3116,7 @@ dependencies = [ "jsonrpc-core", "lazy_static", "log 0.4.14", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "serde", ] @@ -3150,7 +3150,7 @@ dependencies = [ "jsonrpc-server-utils", "log 0.4.14", "parity-ws", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "slab", ] @@ -3287,7 +3287,7 @@ checksum = "c3b6b85fc643f5acd0bffb2cc8a6d150209379267af0d41db72170021841f9f5" dependencies = [ "kvdb", "parity-util-mem", - "parking_lot 0.11.1", + "parking_lot 0.11.2", ] [[package]] @@ -3302,7 +3302,7 @@ dependencies = [ "num_cpus", "owning_ref", "parity-util-mem", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "regex", "rocksdb", "smallvec 1.7.0", @@ -3419,7 +3419,7 @@ dependencies = [ "libp2p-websocket", "libp2p-yamux", "multiaddr", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project 1.0.8", "smallvec 1.7.0", "wasm-timer", @@ -3444,7 +3444,7 @@ dependencies = [ "multiaddr", "multihash 0.14.0", "multistream-select", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project 1.0.8", "prost 0.8.0", "prost-build 0.8.0", @@ -3603,7 +3603,7 @@ dependencies = [ "libp2p-core", "log 0.4.14", "nohash-hasher", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "smallvec 1.7.0", "unsigned-varint 0.7.0", @@ -3815,7 +3815,7 @@ checksum = "214cc0dd9c37cbed27f0bb1eba0c41bbafdb93a8be5e9d6ae1e6b4b42cd044bf" dependencies = [ "futures 0.3.16", "libp2p-core", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "thiserror", "yamux", ] @@ -4000,9 +4000,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" dependencies = [ "scopeguard", ] @@ -5501,6 +5501,30 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-election-playground" +version = "4.0.0-dev" +dependencies = [ + "frame-election-provider-support", + "frame-support", + "frame-system", + "pallet-bags-list", + "pallet-balances", + "pallet-election-provider-multi-phase", + "pallet-session", + "pallet-staking", + "pallet-staking-reward-curve", + "pallet-timestamp", + "parity-scale-codec", + "parking_lot 0.11.2", + "scale-info", + "sp-core", + "sp-io", + "sp-npos-elections", + "sp-runtime", + "sp-tracing", +] + [[package]] name = "pallet-election-provider-multi-phase" version = "4.0.0-dev" @@ -5512,7 +5536,7 @@ dependencies = [ "log 0.4.14", "pallet-balances", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "scale-info", "sp-arithmetic", @@ -6285,7 +6309,7 @@ dependencies = [ "log 0.4.14", "lz4", "memmap2 0.2.1", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.8.4", "snap", ] @@ -6346,7 +6370,7 @@ dependencies = [ "hashbrown 0.11.2", "impl-trait-for-tuples", "parity-util-mem-derive", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "primitive-types", "smallvec 1.7.0", "winapi 0.3.9", @@ -6415,13 +6439,13 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", - "lock_api 0.4.2", - "parking_lot_core 0.8.3", + "lock_api 0.4.5", + "parking_lot_core 0.8.5", ] [[package]] @@ -6441,14 +6465,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.2.5", + "redox_syscall 0.2.10", "smallvec 1.7.0", "winapi 0.3.9", ] @@ -6843,7 +6867,7 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "thiserror", ] @@ -7290,9 +7314,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.5" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ "bitflags", ] @@ -7304,7 +7328,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ "getrandom 0.2.3", - "redox_syscall 0.2.5", + "redox_syscall 0.2.10", ] [[package]] @@ -7657,7 +7681,7 @@ dependencies = [ "futures-timer 3.0.2", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-block-builder", "sc-client-api", "sc-proposer-metrics", @@ -7763,7 +7787,7 @@ dependencies = [ "hash-db", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-executor", "sc-transaction-pool-api", "sc-utils", @@ -7796,7 +7820,7 @@ dependencies = [ "log 0.4.14", "parity-db", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "quickcheck", "sc-client-api", "sc-state-db", @@ -7821,7 +7845,7 @@ dependencies = [ "futures-timer 3.0.2", "libp2p", "log 0.4.14", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-client-api", "sc-utils", "serde", @@ -7846,7 +7870,7 @@ dependencies = [ "getrandom 0.2.3", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-block-builder", "sc-client-api", "sc-consensus", @@ -7888,7 +7912,7 @@ dependencies = [ "num-rational 0.2.4", "num-traits", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "rand_chacha 0.2.2", "retain_mut", @@ -8012,7 +8036,7 @@ dependencies = [ "futures-timer 3.0.2", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-client-api", "sc-consensus", "sp-api", @@ -8071,7 +8095,7 @@ dependencies = [ "libsecp256k1 0.7.0", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "paste 1.0.6", "regex", "sc-executor-common", @@ -8168,7 +8192,7 @@ dependencies = [ "futures-timer 3.0.2", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.8.4", "sc-block-builder", "sc-client-api", @@ -8247,7 +8271,7 @@ dependencies = [ "async-trait", "derive_more", "hex", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "serde_json", "sp-application-crypto", "sp-core", @@ -8280,7 +8304,7 @@ dependencies = [ "log 0.4.14", "lru 0.7.0", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project 1.0.8", "prost 0.9.0", "prost-build 0.9.0", @@ -8340,7 +8364,7 @@ dependencies = [ "futures-timer 3.0.2", "libp2p", "log 0.4.14", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "sc-block-builder", "sc-client-api", @@ -8372,7 +8396,7 @@ dependencies = [ "num_cpus", "once_cell", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "sc-block-builder", "sc-client-api", @@ -8426,7 +8450,7 @@ dependencies = [ "lazy_static", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-block-builder", "sc-chain-spec", "sc-client-api", @@ -8462,7 +8486,7 @@ dependencies = [ "jsonrpc-pubsub", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-chain-spec", "sc-transaction-pool-api", "serde", @@ -8521,7 +8545,7 @@ dependencies = [ "log 0.4.14", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project 1.0.8", "rand 0.7.3", "sc-block-builder", @@ -8581,7 +8605,7 @@ dependencies = [ "hex-literal", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-block-builder", "sc-client-api", "sc-client-db", @@ -8615,7 +8639,7 @@ dependencies = [ "parity-scale-codec", "parity-util-mem", "parity-util-mem-derive", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-client-api", "sp-core", ] @@ -8649,7 +8673,7 @@ dependencies = [ "futures 0.3.16", "libp2p", "log 0.4.14", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project 1.0.8", "rand 0.7.3", "serde", @@ -8670,7 +8694,7 @@ dependencies = [ "libc", "log 0.4.14", "once_cell", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "regex", "rustc-hash", "sc-client-api", @@ -8712,7 +8736,7 @@ dependencies = [ "log 0.4.14", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "retain_mut", "sc-block-builder", "sc-client-api", @@ -9324,7 +9348,7 @@ dependencies = [ "log 0.4.14", "lru 0.7.0", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sp-api", "sp-consensus", "sp-database", @@ -9448,7 +9472,7 @@ dependencies = [ "num-traits", "parity-scale-codec", "parity-util-mem", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "primitive-types", "rand 0.7.3", "regex", @@ -9503,7 +9527,7 @@ name = "sp-database" version = "4.0.0-dev" dependencies = [ "kvdb", - "parking_lot 0.11.1", + "parking_lot 0.11.2", ] [[package]] @@ -9565,7 +9589,7 @@ dependencies = [ "libsecp256k1 0.7.0", "log 0.4.14", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sp-core", "sp-externalities", "sp-keystore", @@ -9598,7 +9622,7 @@ dependencies = [ "futures 0.3.16", "merlin", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.7.3", "rand_chacha 0.2.2", "schnorrkel", @@ -9838,7 +9862,7 @@ dependencies = [ "log 0.4.14", "num-traits", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pretty_assertions", "rand 0.7.3", "smallvec 1.7.0", @@ -10290,7 +10314,7 @@ dependencies = [ "derive_more", "futures 0.3.16", "parity-scale-codec", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "sc-transaction-pool", "sc-transaction-pool-api", "sp-blockchain", @@ -10398,7 +10422,7 @@ dependencies = [ "cfg-if 1.0.0", "libc", "rand 0.8.4", - "redox_syscall 0.2.5", + "redox_syscall 0.2.10", "remove_dir_all", "winapi 0.3.9", ] @@ -10603,7 +10627,7 @@ dependencies = [ "mio 0.7.13", "num_cpus", "once_cell", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-project-lite 0.2.6", "signal-hook-registry", "tokio-macros", @@ -10842,7 +10866,7 @@ dependencies = [ "chrono", "lazy_static", "matchers", - "parking_lot 0.11.1", + "parking_lot 0.9.0", "regex", "serde", "serde_json", @@ -10951,7 +10975,7 @@ dependencies = [ "lazy_static", "log 0.4.14", "lru-cache", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "resolv-conf", "smallvec 1.7.0", "thiserror", @@ -11350,7 +11374,7 @@ checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" dependencies = [ "futures 0.3.16", "js-sys", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "pin-utils", "wasm-bindgen", "wasm-bindgen-futures", @@ -11946,7 +11970,7 @@ dependencies = [ "futures 0.3.16", "log 0.4.14", "nohash-hasher", - "parking_lot 0.11.1", + "parking_lot 0.11.2", "rand 0.8.4", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index f30b223a9b205..dcb72b2283398 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,7 @@ members = [ "frame/elections", "frame/election-provider-multi-phase", "frame/election-provider-support", + "frame/election-playground", "frame/examples/basic", "frame/examples/offchain-worker", "frame/examples/parallel", diff --git a/frame/election-playground/Cargo.toml b/frame/election-playground/Cargo.toml new file mode 100644 index 0000000000000..77b188258bcec --- /dev/null +++ b/frame/election-playground/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "pallet-election-playground" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "election playground" +readme = "README.md" +publish = false + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } +scale-info = { package = "scale-info", version = "1.0.0", features = ["derive"] } + +sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } +sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections" } + +frame-system = { version = "4.0.0-dev", path = "../system" } +frame-support = { version = "4.0.0-dev", path = "../support" } +frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } + +pallet-balances = { version = "4.0.0-dev", path = "../balances" } +pallet-bags-list = { version = "4.0.0-dev", path = "../bags-list" } +pallet-staking = { version = "4.0.0-dev", path = "../staking" } +pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } +pallet-session = { version = "4.0.0-dev", path = "../session" } +pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } +pallet-election-provider-multi-phase = { version = "4.0.0-dev", path = "../election-provider-multi-phase" } +parking_lot = { version = "0.11.2" } + + +[dev-dependencies] + +sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } +sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } + diff --git a/frame/election-playground/src/lib.rs b/frame/election-playground/src/lib.rs new file mode 100644 index 0000000000000..0d1df7c99a500 --- /dev/null +++ b/frame/election-playground/src/lib.rs @@ -0,0 +1,65 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! A pallet that adds no functionality but only provides end-to-end tests for the election +//! pipeline. +//! +//! Some tests are written as standalone tests using the [`mock`] module as the runtime. Some will +//! be written such that they can be used against any runtime. In other words, anu runtime that +//! fulfills [`ElectionRuntime`]. +//! +//! It will also expose some functions as runtime APIs. This allows a test client (e.g. try-runtime) +//! to call into these functions from wasm. + +use frame_support::pallet_prelude::*; +use sp_runtime::traits::One; + +#[cfg(test)] +mod mock; + +pub trait ElectionRuntime: + pallet_election_provider_multi_phase::Config + pallet_staking::Config + pallet_bags_list::Config +{ +} +impl< + T: pallet_election_provider_multi_phase::Config + + pallet_staking::Config + + pallet_bags_list::Config, + > ElectionRuntime for T +{ +} + +pub fn roll_to(n: T::BlockNumber) { + let now = frame_system::Pallet::::block_number(); + for i in now + One::one()..=n { + frame_system::Pallet::::set_block_number(i); + pallet_bags_list::Pallet::::on_initialize(i); + pallet_election_provider_multi_phase::Pallet::::on_initialize(i); + pallet_staking::Pallet::::on_initialize(i); + } +} + +pub fn roll_to_with_ocw(n: T::BlockNumber) { + let now = frame_system::Pallet::::block_number(); + for i in now + One::one()..=n { + frame_system::Pallet::::set_block_number(i); + pallet_bags_list::Pallet::::on_initialize(i); + pallet_election_provider_multi_phase::Pallet::::on_initialize(i); + pallet_election_provider_multi_phase::Pallet::::offchain_worker(i); + pallet_staking::Pallet::::on_initialize(i); + } +} diff --git a/frame/election-playground/src/mock.rs b/frame/election-playground/src/mock.rs new file mode 100644 index 0000000000000..f134c0b6e66cb --- /dev/null +++ b/frame/election-playground/src/mock.rs @@ -0,0 +1,315 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::*; +use frame_election_provider_support::{ + data_provider, onchain, ElectionDataProvider, ExtendedBalance, SequentialPhragmen, + SnapshotBounds, +}; +pub use frame_support::{assert_noop, assert_ok}; +use frame_support::{parameter_types, traits::Hooks, weights::Weight}; +use pallet_session::{PeriodicSessions, TestSessionHandler}; +use pallet_staking::{ConvertCurve, EraIndex, SessionInterface}; +use parking_lot::RwLock; +use sp_core::{ + offchain::{ + testing::{PoolState, TestOffchainExt, TestTransactionPoolExt}, + OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, + }, + H256, +}; + +use multi_phase::SolutionAccuracyOf; +use pallet_election_provider_multi_phase as multi_phase; +use sp_runtime::{ + curve::PiecewiseLinear, + testing::Header, + traits::{BlakeTwo256, Bounded, IdentityLookup}, + PerU16, Perbill, +}; +use std::sync::Arc; + +frame_support::construct_runtime!( + pub enum TestRuntime where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic + { + System: frame_system::{Pallet, Call, Config, Storage, Event}, + Balances: pallet_balances::{Pallet, Call, Config, Storage, Event}, + BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, + Timestamp: pallet_timestamp::{Pallet, Call, Storage,}, + Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, + MultiPhase: multi_phase::{Pallet, Call, Storage, Event}, + } +); + +pub(crate) type Balance = u64; +pub(crate) type AccountId = u64; +pub(crate) type BlockNumber = u64; +pub(crate) type VoterIndex = u32; +pub(crate) type TargetIndex = u16; + +impl frame_system::offchain::SendTransactionTypes for TestRuntime +where + Call: From, +{ + type OverarchingCall = Call; + type Extrinsic = Extrinsic; +} + +pub type Extrinsic = sp_runtime::testing::TestXt; +pub type Block = sp_runtime::generic::Block; +pub type UncheckedExtrinsic = sp_runtime::generic::UncheckedExtrinsic; + +const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); +parameter_types! { + pub const ExistentialDeposit: u64 = 1; + pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights + ::with_sensible_defaults(2 * frame_support::weights::constants::WEIGHT_PER_SECOND, NORMAL_DISPATCH_RATIO); +} + +impl frame_system::Config for TestRuntime { + type SS58Prefix = (); + type BaseCallFilter = frame_support::traits::Everything; + type Origin = Origin; + type Index = u64; + type BlockNumber = BlockNumber; + type Call = Call; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = AccountId; + type Lookup = IdentityLookup; + type Header = Header; + type Event = Event; + type BlockHashCount = (); + type DbWeight = (); + type BlockLength = (); + type BlockWeights = BlockWeights; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = pallet_balances::AccountData; + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type OnSetCode = (); +} + +impl pallet_balances::Config for TestRuntime { + type Balance = Balance; + type Event = Event; + type DustRemoval = (); + type ExistentialDeposit = ExistentialDeposit; + type AccountStore = System; + type MaxLocks = (); + type MaxReserves = (); + type ReserveIdentifier = [u8; 8]; + type WeightInfo = (); +} + +parameter_types! { + pub static SignedPhase: BlockNumber = 10; + pub static UnsignedPhase: BlockNumber = 5; + pub static SignedMaxSubmissions: u32 = 5; + pub static SignedDepositBase: Balance = 5; + pub static SignedDepositByte: Balance = 0; + pub static SignedDepositWeight: Balance = 0; + pub static SignedRewardBase: Balance = 7; + pub static SignedMaxWeight: Weight = BlockWeights::get().max_block; + pub static MinerTxPriority: u64 = 100; + pub static SolutionImprovementThreshold: Perbill = Perbill::zero(); + pub static OffchainRepeat: BlockNumber = 5; + pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; + pub static MinerMaxLength: u32 = 256; + pub static MockWeightInfo: bool = false; + pub static VoterSnapshotBounds: SnapshotBounds = SnapshotBounds::new_unbounded(); + pub static TargetSnapshotBounds: SnapshotBounds = SnapshotBounds::new_unbounded(); + + pub static EpochLength: u64 = 30; + pub static OnChianFallback: bool = true; + pub static Balancing: Option<(usize, ExtendedBalance)> = None; +} + +impl onchain::Config for TestRuntime { + type Accuracy = sp_runtime::Perbill; + type DataProvider = Staking; +} + +sp_npos_elections::generate_solution_type!( + #[compact] + pub struct TestNposSolution::(16) +); + +impl pallet_election_provider_multi_phase::Config for TestRuntime { + type Event = Event; + type Currency = Balances; + type EstimateCallFee = frame_support::traits::ConstU32<8>; + type SignedPhase = SignedPhase; + type UnsignedPhase = UnsignedPhase; + type SolutionImprovementThreshold = SolutionImprovementThreshold; + type OffchainRepeat = OffchainRepeat; + type MinerMaxWeight = MinerMaxWeight; + type MinerMaxLength = MinerMaxLength; + type MinerTxPriority = MinerTxPriority; + type SignedRewardBase = SignedRewardBase; + type SignedDepositBase = SignedDepositBase; + type SignedDepositByte = (); + type SignedDepositWeight = (); + type SignedMaxWeight = SignedMaxWeight; + type SignedMaxSubmissions = SignedMaxSubmissions; + type SlashHandler = (); + type RewardHandler = (); + type DataProvider = Staking; + type WeightInfo = (); + type BenchmarkingConfig = (); + type Fallback = frame_election_provider_support::onchain::OnChainSequentialPhragmen; + type ForceOrigin = frame_system::EnsureRoot; + type Solution = TestNposSolution; + type VoterSnapshotBounds = VoterSnapshotBounds; + type TargetSnapshotBounds = TargetSnapshotBounds; + type Solver = SequentialPhragmen, Balancing>; +} + +parameter_types! { + pub const MinimumPeriod: u64 = 5; +} +impl pallet_timestamp::Config for TestRuntime { + type Moment = u64; + type OnTimestampSet = (); + type MinimumPeriod = MinimumPeriod; + type WeightInfo = (); +} + +pallet_staking_reward_curve::build! { + const I_NPOS: PiecewiseLinear<'static> = curve!( + min_inflation: 0_025_000, + max_inflation: 0_100_000, + ideal_stake: 0_500_000, + falloff: 0_050_000, + max_piece_count: 40, + test_precision: 0_005_000, + ); +} + +parameter_types! { + pub static BondingDuration: EraIndex = 5; + pub static SessionsPerEra: u32 = 5; + pub static SlashDeferDuration: EraIndex = 0; + pub static MaxNominatorRewardedPerValidator: u32 = 10; + pub static OffendingValidatorsThreshold: Perbill = Perbill::from_percent(50); + pub const RewardCurve: &'static PiecewiseLinear<'static> = &I_NPOS; + + pub static SessionOffset: BlockNumber = 0; + pub static SessionPeriod: BlockNumber = 10; +} + +struct TestSessionInterface; +impl SessionInterface for TestSessionInterface { + fn disable_validator(validator_index: u32) -> bool { + Default::default() + } + + fn validators() -> Vec { + Default::default() + } + + fn prune_historical_up_to(up_to: SessionIndex) {} +} + +impl pallet_staking::Config for TestRuntime { + type Currency = Balances; + type UnixTime = Timestamp; + type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; + type RewardRemainder = (); + type Event = Event; + type Slash = (); + type Reward = (); + type SessionsPerEra = SessionsPerEra; + type SlashDeferDuration = SlashDeferDuration; + type SlashCancelOrigin = frame_system::EnsureRoot; + type BondingDuration = BondingDuration; + type SessionInterface = TestSessionInterface; + type EraPayout = ConvertCurve; + type NextNewSession = PeriodicSessions; + type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; + type OffendingValidatorsThreshold = OffendingValidatorsThreshold; + type ElectionProvider = MultiPhase; + type GenesisElectionProvider = onchain::OnChainSequentialPhragmen; + type SortedListProvider = BagsList; + type NominationQuota = pallet_staking::FixedNominationQuota<16>; + type WeightInfo = (); +} + +const THRESHOLDS: [sp_npos_elections::VoteWeight; 9] = + [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; + +parameter_types! { + pub static BagThresholds: &'static [sp_npos_elections::VoteWeight] = &THRESHOLDS; +} + +impl pallet_bags_list::Config for TestRuntime { + type Event = Event; + type WeightInfo = (); + type VoteWeightProvider = Staking; + type BagThresholds = BagThresholds; +} + +#[derive(Default)] +pub struct ExtBuilder {} + +impl ExtBuilder { + pub fn build(self) -> sp_io::TestExternalities { + sp_tracing::try_init_simple(); + let mut storage = + frame_system::GenesisConfig::default().build_storage::().unwrap(); + + let _ = pallet_balances::GenesisConfig:: { + balances: vec![ + // bunch of account for submitting stuff only. + (99, 100), + (999, 100), + (9999, 100), + ], + } + .assimilate_storage(&mut storage); + + sp_io::TestExternalities::from(storage) + } + + pub fn build_offchainify( + self, + iters: u32, + ) -> (sp_io::TestExternalities, Arc>) { + let mut ext = self.build(); + let (offchain, offchain_state) = TestOffchainExt::new(); + let (pool, pool_state) = TestTransactionPoolExt::new(); + + let mut seed = [0_u8; 32]; + seed[0..4].copy_from_slice(&iters.to_le_bytes()); + offchain_state.write().seed = seed; + + ext.register_extension(OffchainDbExt::new(offchain.clone())); + ext.register_extension(OffchainWorkerExt::new(offchain)); + ext.register_extension(TransactionPoolExt::new(pool)); + + (ext, pool_state) + } + + pub fn build_and_execute(self, test: impl FnOnce() -> ()) { + self.build().execute_with(test) + } +} diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 7e00b0120e3a2..7cb59094dc936 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -890,7 +890,8 @@ impl Pallet { }; for (next, _) in Validators::::iter() { - // TODO: rather sub-optimal, we should not need to track size if it is not bounded. + // NOTE: rather sub-optimal, we should not need to track size if it is not bounded, but + // in this case we prefer not cluttering the code. let new_internal_size = internal_size + sp_std::mem::size_of::(); let new_final_size = new_internal_size + StaticSizeTracker::::length_prefix(targets.len() + 1); @@ -1503,16 +1504,9 @@ impl StaticSizeTracker { /// The length prefix of a vector with the given length. #[inline] pub(crate) fn length_prefix(length: usize) -> usize { - // TODO: scale codec could and should expose a public function for this that I can reuse. - match length { - 0..=63 => 1, - 64..=16383 => 2, - 16384..=1073741823 => 4, - // this arm almost always never happens. Although, it would be good to get rid of of it, - // for otherwise we could make this function const, which might enable further - // optimizations. - x @ _ => codec::Compact(x as u32).encoded_size(), - } + use codec::{Compact, CompactLen}; + let length = length as u32; + Compact::::compact_len(&length) } /// Register a voter in `self` who has casted `votes`. From a60ca9fc9d7cb084a063414e10556135869a3362 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Wed, 24 Nov 2021 10:35:48 +0100 Subject: [PATCH 6/7] new test commit to enable signing --- frame/election-playground/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-playground/src/lib.rs b/frame/election-playground/src/lib.rs index 0d1df7c99a500..f6db9ef2d3325 100644 --- a/frame/election-playground/src/lib.rs +++ b/frame/election-playground/src/lib.rs @@ -43,7 +43,7 @@ impl< { } -pub fn roll_to(n: T::BlockNumber) { +pub fn roll_to(n: T::BlockNumber, foo: u32) { let now = frame_system::Pallet::::block_number(); for i in now + One::one()..=n { frame_system::Pallet::::set_block_number(i); From 95bbaf24f68f2ca1efcb472c46fcafe462a1850a Mon Sep 17 00:00:00 2001 From: kianenigma Date: Tue, 30 Nov 2021 09:13:24 +0100 Subject: [PATCH 7/7] remove election playground stuff' --- Cargo.toml | 1 - frame/election-playground/Cargo.toml | 43 ---- frame/election-playground/src/lib.rs | 76 ------ frame/election-playground/src/mock.rs | 325 -------------------------- 4 files changed, 445 deletions(-) delete mode 100644 frame/election-playground/Cargo.toml delete mode 100644 frame/election-playground/src/lib.rs delete mode 100644 frame/election-playground/src/mock.rs diff --git a/Cargo.toml b/Cargo.toml index dcb72b2283398..f30b223a9b205 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,6 @@ members = [ "frame/elections", "frame/election-provider-multi-phase", "frame/election-provider-support", - "frame/election-playground", "frame/examples/basic", "frame/examples/offchain-worker", "frame/examples/parallel", diff --git a/frame/election-playground/Cargo.toml b/frame/election-playground/Cargo.toml deleted file mode 100644 index 024e8db336055..0000000000000 --- a/frame/election-playground/Cargo.toml +++ /dev/null @@ -1,43 +0,0 @@ -[package] -name = "pallet-election-playground" -version = "4.0.0-dev" -authors = ["Parity Technologies "] -edition = "2021" -license = "Apache-2.0" -homepage = "https://substrate.io" -repository = "https://github.com/paritytech/substrate/" -description = "election playground" -readme = "README.md" -publish = false - -[package.metadata.docs.rs] -targets = ["x86_64-unknown-linux-gnu"] - -[dependencies] -codec = { package = "parity-scale-codec", version = "2.0.0", features = ["derive"] } -scale-info = { package = "scale-info", version = "1.0.0", features = ["derive"] } - -sp-runtime = { version = "4.0.0-dev", path = "../../primitives/runtime" } -sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections" } -sp-staking = { version = "4.0.0-dev", path = "../../primitives/staking" } - -frame-system = { version = "4.0.0-dev", path = "../system" } -frame-support = { version = "4.0.0-dev", path = "../support" } -frame-election-provider-support = { version = "4.0.0-dev", path = "../election-provider-support" } - -pallet-balances = { version = "4.0.0-dev", path = "../balances" } -pallet-bags-list = { version = "4.0.0-dev", path = "../bags-list" } -pallet-staking = { version = "4.0.0-dev", path = "../staking" } -pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } -pallet-session = { version = "4.0.0-dev", path = "../session" } -pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } -pallet-election-provider-multi-phase = { version = "4.0.0-dev", path = "../election-provider-multi-phase" } -parking_lot = { version = "0.11.2" } - - -[dev-dependencies] - -sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } -sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } -sp-io = { version = "4.0.0-dev", path = "../../primitives/io" } - diff --git a/frame/election-playground/src/lib.rs b/frame/election-playground/src/lib.rs deleted file mode 100644 index 6426deabad51a..0000000000000 --- a/frame/election-playground/src/lib.rs +++ /dev/null @@ -1,76 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! A pallet that adds no functionality but only provides end-to-end tests for the election -//! pipeline. -//! -//! Some tests are written as standalone tests using the [`mock`] module as the runtime. Some will -//! be written such that they can be used against any runtime. In other words, anu runtime that -//! fulfills [`ElectionRuntime`]. -//! -//! It will also expose some functions as runtime APIs. This allows a test client (e.g. try-runtime) -//! to call into these functions from wasm. - -use frame_support::pallet_prelude::*; -use sp_runtime::traits::One; - -#[cfg(test)] -mod mock; - -pub trait ElectionRuntime: - pallet_election_provider_multi_phase::Config + pallet_staking::Config + pallet_bags_list::Config -{ -} -impl< - T: pallet_election_provider_multi_phase::Config - + pallet_staking::Config - + pallet_bags_list::Config, - > ElectionRuntime for T -{ -} - -pub fn roll_to(n: T::BlockNumber) { - let mut now = frame_system::Pallet::::block_number(); - while now + One::one() <= n { - now += One::one(); - pallet_bags_list::Pallet::::on_initialize(now); - pallet_election_provider_multi_phase::Pallet::::on_initialize(now); - pallet_staking::Pallet::::on_initialize(now); - } -} - -pub fn roll_to_with_ocw(n: T::BlockNumber) { - let mut now = frame_system::Pallet::::block_number(); - while now + One::one() <= n { - now += One::one(); - frame_system::Pallet::::set_block_number(now); - pallet_bags_list::Pallet::::on_initialize(now); - pallet_election_provider_multi_phase::Pallet::::on_initialize(now); - pallet_election_provider_multi_phase::Pallet::::offchain_worker(now); - pallet_staking::Pallet::::on_initialize(now); - } -} - -/// Simple test demonstrating what happens in the staking system, end to end. -pub fn simple_end_to_end() { - // some data must currently exist in pallet-staking. We first fast-forward to the corresponding - // block of the next election - let now = frame_system::Pallet::::block_number(); - let next_election = - ::NextNewSession::estimate_next_new_session(now); - roll_to::(next_election); -} diff --git a/frame/election-playground/src/mock.rs b/frame/election-playground/src/mock.rs deleted file mode 100644 index 825bea92aefc5..0000000000000 --- a/frame/election-playground/src/mock.rs +++ /dev/null @@ -1,325 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use frame_election_provider_support::{ - onchain, ExtendedBalance, SequentialPhragmen, SnapshotBounds, -}; -pub use frame_support::{assert_noop, assert_ok}; -use frame_support::{parameter_types, traits::Hooks, weights::Weight}; -use pallet_session::{PeriodicSessions, TestSessionHandler}; -use pallet_staking::{ConvertCurve, EraIndex, SessionInterface}; -use parking_lot::RwLock; -use sp_core::{ - offchain::{ - testing::{PoolState, TestOffchainExt, TestTransactionPoolExt}, - OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, - }, - H256, -}; - -use multi_phase::SolutionAccuracyOf; -use pallet_election_provider_multi_phase as multi_phase; -use sp_runtime::{ - curve::PiecewiseLinear, - testing::Header, - traits::{BlakeTwo256, Bounded, IdentityLookup, OpaqueKeys}, - PerU16, Perbill, -}; -use std::sync::Arc; - -frame_support::construct_runtime!( - pub enum TestRuntime where - Block = Block, - NodeBlock = Block, - UncheckedExtrinsic = UncheckedExtrinsic - { - System: frame_system::{Pallet, Call, Config, Storage, Event}, - Balances: pallet_balances::{Pallet, Call, Config, Storage, Event}, - BagsList: pallet_bags_list::{Pallet, Call, Storage, Event}, - Timestamp: pallet_timestamp::{Pallet, Call, Storage,}, - Session: pallet_session::{Pallet, Call, Storage, Config, Event}, - Staking: pallet_staking::{Pallet, Call, Config, Storage, Event}, - MultiPhase: multi_phase::{Pallet, Call, Storage, Event}, - } -); - -pub(crate) type Balance = u64; -pub(crate) type AccountId = u64; -pub(crate) type BlockNumber = u64; -pub(crate) type VoterIndex = u32; -pub(crate) type TargetIndex = u16; - -impl frame_system::offchain::SendTransactionTypes for TestRuntime -where - Call: From, -{ - type OverarchingCall = Call; - type Extrinsic = Extrinsic; -} - -pub type Extrinsic = sp_runtime::testing::TestXt; -pub type Block = sp_runtime::generic::Block; -pub type UncheckedExtrinsic = sp_runtime::generic::UncheckedExtrinsic; - -const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); -parameter_types! { - pub const ExistentialDeposit: u64 = 1; - pub BlockWeights: frame_system::limits::BlockWeights = frame_system::limits::BlockWeights - ::with_sensible_defaults(2 * frame_support::weights::constants::WEIGHT_PER_SECOND, NORMAL_DISPATCH_RATIO); -} - -impl frame_system::Config for TestRuntime { - type SS58Prefix = (); - type BaseCallFilter = frame_support::traits::Everything; - type Origin = Origin; - type Index = u64; - type BlockNumber = BlockNumber; - type Call = Call; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = AccountId; - type Lookup = IdentityLookup; - type Header = Header; - type Event = Event; - type BlockHashCount = (); - type DbWeight = (); - type BlockLength = (); - type BlockWeights = BlockWeights; - type Version = (); - type PalletInfo = PalletInfo; - type AccountData = pallet_balances::AccountData; - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type OnSetCode = (); -} - -impl pallet_balances::Config for TestRuntime { - type Balance = Balance; - type Event = Event; - type DustRemoval = (); - type ExistentialDeposit = ExistentialDeposit; - type AccountStore = System; - type MaxLocks = (); - type MaxReserves = (); - type ReserveIdentifier = [u8; 8]; - type WeightInfo = (); -} - -parameter_types! { - pub static SignedPhase: BlockNumber = 10; - pub static UnsignedPhase: BlockNumber = 5; - pub static SignedMaxSubmissions: u32 = 5; - pub static SignedDepositBase: Balance = 5; - pub static SignedDepositByte: Balance = 0; - pub static SignedDepositWeight: Balance = 0; - pub static SignedRewardBase: Balance = 7; - pub static SignedMaxWeight: Weight = BlockWeights::get().max_block; - pub static MinerTxPriority: u64 = 100; - pub static SolutionImprovementThreshold: Perbill = Perbill::zero(); - pub static OffchainRepeat: BlockNumber = 5; - pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; - pub static MinerMaxLength: u32 = 256; - pub static MockWeightInfo: bool = false; - pub static VoterSnapshotBounds: SnapshotBounds = SnapshotBounds::new_unbounded(); - pub static TargetSnapshotBounds: SnapshotBounds = SnapshotBounds::new_unbounded(); - - pub static EpochLength: u64 = 30; - pub static OnChianFallback: bool = true; - pub static Balancing: Option<(usize, ExtendedBalance)> = None; -} - -impl onchain::Config for TestRuntime { - type Accuracy = sp_runtime::Perbill; - type DataProvider = Staking; -} - -sp_npos_elections::generate_solution_type!( - #[compact] - pub struct TestNposSolution::(16) -); - -impl pallet_election_provider_multi_phase::Config for TestRuntime { - type Event = Event; - type Currency = Balances; - type EstimateCallFee = frame_support::traits::ConstU32<8>; - type SignedPhase = SignedPhase; - type UnsignedPhase = UnsignedPhase; - type SolutionImprovementThreshold = SolutionImprovementThreshold; - type OffchainRepeat = OffchainRepeat; - type MinerMaxWeight = MinerMaxWeight; - type MinerMaxLength = MinerMaxLength; - type MinerTxPriority = MinerTxPriority; - type SignedRewardBase = SignedRewardBase; - type SignedDepositBase = SignedDepositBase; - type SignedDepositByte = (); - type SignedDepositWeight = (); - type SignedMaxWeight = SignedMaxWeight; - type SignedMaxSubmissions = SignedMaxSubmissions; - type SlashHandler = (); - type RewardHandler = (); - type DataProvider = Staking; - type WeightInfo = (); - type BenchmarkingConfig = pallet_election_provider_multi_phase::TestBenchmarkingConfig; - type Fallback = frame_election_provider_support::onchain::OnChainSequentialPhragmen; - type ForceOrigin = frame_system::EnsureRoot; - type Solution = TestNposSolution; - type VoterSnapshotBounds = VoterSnapshotBounds; - type TargetSnapshotBounds = TargetSnapshotBounds; - type Solver = SequentialPhragmen, Balancing>; -} - -parameter_types! { - pub const MinimumPeriod: u64 = 5; -} -impl pallet_timestamp::Config for TestRuntime { - type Moment = u64; - type OnTimestampSet = (); - type MinimumPeriod = MinimumPeriod; - type WeightInfo = (); -} - -pallet_staking_reward_curve::build! { - const I_NPOS: PiecewiseLinear<'static> = curve!( - min_inflation: 0_025_000, - max_inflation: 0_100_000, - ideal_stake: 0_500_000, - falloff: 0_050_000, - max_piece_count: 40, - test_precision: 0_005_000, - ); -} - -parameter_types! { - pub static BondingDuration: EraIndex = 5; - pub static SessionsPerEra: u32 = 5; - pub static SlashDeferDuration: EraIndex = 0; - pub static MaxNominatorRewardedPerValidator: u32 = 10; - pub static OffendingValidatorsThreshold: Perbill = Perbill::from_percent(50); - pub const RewardCurve: &'static PiecewiseLinear<'static> = &I_NPOS; - - pub static SessionOffset: BlockNumber = 0; - pub static SessionPeriod: BlockNumber = 10; -} - -sp_runtime::impl_opaque_keys! { - pub struct MockSessionKeys { - pub foo: sp_runtime::testing::UintAuthorityId, - } -} - -impl pallet_session::historical::Config for TestRuntime { - type FullIdentification = pallet_staking::Exposure; - type FullIdentificationOf = pallet_staking::ExposureOf; -} - -impl pallet_session::Config for TestRuntime { - type Event = Event; - type ValidatorId = AccountId; - type ValidatorIdOf = pallet_staking::StashOf; - type ShouldEndSession = pallet_session::PeriodicSessions; - type NextSessionRotation = pallet_session::PeriodicSessions; - type SessionManager = Staking; - type SessionHandler = TestSessionHandler; - type Keys = MockSessionKeys; - type WeightInfo = (); -} - -impl pallet_staking::Config for TestRuntime { - type Currency = Balances; - type UnixTime = Timestamp; - type CurrencyToVote = frame_support::traits::SaturatingCurrencyToVote; - type RewardRemainder = (); - type Event = Event; - type Slash = (); - type Reward = (); - type SessionsPerEra = SessionsPerEra; - type SlashDeferDuration = SlashDeferDuration; - type SlashCancelOrigin = frame_system::EnsureRoot; - type BondingDuration = BondingDuration; - type SessionInterface = Self; - type EraPayout = ConvertCurve; - type NextNewSession = Session; - type MaxNominatorRewardedPerValidator = MaxNominatorRewardedPerValidator; - type OffendingValidatorsThreshold = OffendingValidatorsThreshold; - type ElectionProvider = MultiPhase; - type GenesisElectionProvider = onchain::OnChainSequentialPhragmen; - type SortedListProvider = BagsList; - type NominationQuota = pallet_staking::FixedNominationQuota<16>; - type BenchmarkingConfig = pallet_staking::TestBenchmarkingConfig; - type WeightInfo = (); -} - -const THRESHOLDS: [sp_npos_elections::VoteWeight; 9] = - [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; - -parameter_types! { - pub static BagThresholds: &'static [sp_npos_elections::VoteWeight] = &THRESHOLDS; -} - -impl pallet_bags_list::Config for TestRuntime { - type Event = Event; - type WeightInfo = (); - type VoteWeightProvider = Staking; - type BagThresholds = BagThresholds; -} - -#[derive(Default)] -pub struct ExtBuilder {} - -impl ExtBuilder { - pub fn build(self) -> sp_io::TestExternalities { - sp_tracing::try_init_simple(); - let mut storage = - frame_system::GenesisConfig::default().build_storage::().unwrap(); - - let _ = pallet_balances::GenesisConfig:: { - balances: vec![ - // bunch of account for submitting stuff only. - (99, 100), - (999, 100), - (9999, 100), - ], - } - .assimilate_storage(&mut storage); - - sp_io::TestExternalities::from(storage) - } - - pub fn build_offchainify( - self, - iters: u32, - ) -> (sp_io::TestExternalities, Arc>) { - let mut ext = self.build(); - let (offchain, offchain_state) = TestOffchainExt::new(); - let (pool, pool_state) = TestTransactionPoolExt::new(); - - let mut seed = [0_u8; 32]; - seed[0..4].copy_from_slice(&iters.to_le_bytes()); - offchain_state.write().seed = seed; - - ext.register_extension(OffchainDbExt::new(offchain.clone())); - ext.register_extension(OffchainWorkerExt::new(offchain)); - ext.register_extension(TransactionPoolExt::new(pool)); - - (ext, pool_state) - } - - pub fn build_and_execute(self, test: impl FnOnce() -> ()) { - self.build().execute_with(test) - } -}