From a549691a57604ee311997a66c81583a0fa9dc771 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 14:15:19 +0200 Subject: [PATCH 01/82] start sketching out how voter bags might be implemented --- frame/staking/src/lib.rs | 31 +++++++++- frame/staking/src/voter_bags.rs | 101 ++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 3 deletions(-) create mode 100644 frame/staking/src/voter_bags.rs diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 888601e307f35..42dc062569e27 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -276,8 +276,9 @@ pub mod testing_utils; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod benchmarking; -pub mod slashing; pub mod inflation; +pub mod slashing; +pub mod voter_bags; pub mod weights; use sp_std::{ @@ -353,6 +354,9 @@ type NegativeImbalanceOf = <::Currency as Currency< ::AccountId, >>::NegativeImbalance; +type AccountIdOf = ::AccountId; +type VotingDataOf = (AccountIdOf, VoteWeight, Vec>); + /// Information regarding the active era (era in used in session). #[derive(Encode, Decode, RuntimeDebug)] pub struct ActiveEraInfo { @@ -994,6 +998,25 @@ decl_storage! { /// /// This is set to v6.0.0 for new networks. StorageVersion build(|_: &GenesisConfig| Releases::V6_0_0): Releases; + + // The next storage items collectively comprise the voter bags: a composite data structure + // designed to allow efficient iteration of the top N voters by stake, mostly. See + // `mod voter_bags` for details. + + /// Which bag currently contains a particular voter. + /// + /// This may not be the appropriate bag for the voter's weight if they have been rewarded or + /// slashed. + VoterBagFor: map hasher(twox_64_concat) T::AccountId => voter_bags::BagIdx; + + /// The head and tail of each bag of voters. + VoterBags: map hasher(twox_64_concat) voter_bags::BagIdx => voter_bags::Bag; + + /// The nodes comprising each bag. + VoterNodes: double_map + hasher(twox_64_concat) voter_bags::BagIdx, + hasher(twox_64_concat) T::AccountId => + Option>; } add_extra_genesis { config(stakers): @@ -2495,7 +2518,9 @@ impl Module { /// auto-chilled. /// /// Note that this is VERY expensive. Use with care. - pub fn get_npos_voters() -> Vec<(T::AccountId, VoteWeight, Vec)> { + pub fn get_npos_voters( + maybe_max_len: Option, + ) -> Vec> { let weight_of = Self::slashable_balance_of_fn(); let mut all_voters = Vec::new(); @@ -2561,7 +2586,7 @@ impl frame_election_provider_support::ElectionDataProvider) -> data_provider::Result<(Vec, Weight)> { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs new file mode 100644 index 0000000000000..aa54e61ea5161 --- /dev/null +++ b/frame/staking/src/voter_bags.rs @@ -0,0 +1,101 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implement a data structure designed for the properties that: +//! +//! - It's efficient to insert or remove a voter +//! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of +//! voters doesn't particularly matter. + +use crate::{ + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VotingDataOf, VoteWeight, +}; +use codec::{Encode, Decode}; +use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; +use sp_runtime::SaturatedConversion; +use sp_std::marker::PhantomData; + +/// Index type for a bag. +pub type BagIdx = u8; + +/// Given a certain vote weight, which bag should this voter contain? +fn bag_for(weight: VoteWeight) -> BagIdx { + todo!("geometric series of some description; ask alfonso") +} + +/// Type of voter. +/// +/// Similar to [`crate::StakerStatus`], but somewhat more limited. +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum VoterType { + Validator, + Nominator, +} + +/// Fundamental information about a voter. +#[derive(Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Voter { + /// Account Id of this voter + pub id: AccountId, + /// Whether the voter is a validator or nominator + pub voter_type: VoterType, +} + +pub type VoterOf = Voter>; + +/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. +/// +/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of +/// arbitrary and unbounded length, all having a vote weight within a particular constant range. +/// This structure means that voters can be added and removed in `O(1)` time. +/// +/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. +/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall +/// stake decreases as successive bags are reached. This means that it is valid to truncate +/// iteration at any desired point; only those voters in the lowest bag (who are known to have +/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire +/// for fairness and the requirement for efficiency. +pub struct VoterList(PhantomData); + +pub type VoterListOf = VoterList>; + +/// A Bag contains a singly-linked list of voters. +/// +/// Note that we maintain both head and tail pointers. While it would be possible to get away +/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's +/// more desirable to ensure that there is some element of first-come, first-serve to the list's +/// iteration so that there's no incentive to churn voter positioning to improve the chances of +/// appearing within the voter set. +#[derive(Default, Encode, Decode)] +pub struct Bag { + head: Option, + tail: Option, +} + +pub type BagOf = Bag>; + +/// A Node is the fundamental element comprising the singly-linked lists which for each bag. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Node { + voter: Voter, + next: Option, +} + +pub type NodeOf = Node>; From bdfabd2bd40bba47dad02fef80e44920022be900 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:03:46 +0200 Subject: [PATCH 02/82] storage getters and setters for Bag, Node --- frame/staking/src/lib.rs | 7 +- frame/staking/src/voter_bags.rs | 119 +++++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 34 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 42dc062569e27..7dd8c19cd4487 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1003,6 +1003,9 @@ decl_storage! { // designed to allow efficient iteration of the top N voters by stake, mostly. See // `mod voter_bags` for details. + /// How many voters are registered. + VoterCount: u32; + /// Which bag currently contains a particular voter. /// /// This may not be the appropriate bag for the voter's weight if they have been rewarded or @@ -1010,13 +1013,13 @@ decl_storage! { VoterBagFor: map hasher(twox_64_concat) T::AccountId => voter_bags::BagIdx; /// The head and tail of each bag of voters. - VoterBags: map hasher(twox_64_concat) voter_bags::BagIdx => voter_bags::Bag; + VoterBags: map hasher(twox_64_concat) voter_bags::BagIdx => Option>; /// The nodes comprising each bag. VoterNodes: double_map hasher(twox_64_concat) voter_bags::BagIdx, hasher(twox_64_concat) T::AccountId => - Option>; + Option>; } add_extra_genesis { config(stakers): diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index aa54e61ea5161..8670197398ecf 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,7 +22,7 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VotingDataOf, VoteWeight, + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VoterBagFor, VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; @@ -33,32 +33,15 @@ use sp_std::marker::PhantomData; pub type BagIdx = u8; /// Given a certain vote weight, which bag should this voter contain? -fn bag_for(weight: VoteWeight) -> BagIdx { +fn notional_bag_for(weight: VoteWeight) -> BagIdx { todo!("geometric series of some description; ask alfonso") } -/// Type of voter. -/// -/// Similar to [`crate::StakerStatus`], but somewhat more limited. -#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] -#[cfg_attr(feature = "std", derive(Debug))] -pub enum VoterType { - Validator, - Nominator, -} - -/// Fundamental information about a voter. -#[derive(Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct Voter { - /// Account Id of this voter - pub id: AccountId, - /// Whether the voter is a validator or nominator - pub voter_type: VoterType, +/// Find the actual bag containing the current voter. +fn current_bag_for(id: &AccountIdOf) -> Option { + VoterBagFor::::try_get(id).ok() } -pub type VoterOf = Voter>; - /// Data structure providing efficient mostly-accurate selection of the top N voters by stake. /// /// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of @@ -71,9 +54,13 @@ pub type VoterOf = Voter>; /// iteration at any desired point; only those voters in the lowest bag (who are known to have /// relatively little power to affect the outcome) can be excluded. This satisfies both the desire /// for fairness and the requirement for efficiency. -pub struct VoterList(PhantomData); +pub struct VoterList(PhantomData); -pub type VoterListOf = VoterList>; +impl VoterList { + pub fn decode_len() -> Option { + crate::VoterCount::try_get().ok().map(|n| n.saturated_into()) + } +} /// A Bag contains a singly-linked list of voters. /// @@ -83,19 +70,87 @@ pub type VoterListOf = VoterList>; /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. #[derive(Default, Encode, Decode)] -pub struct Bag { - head: Option, - tail: Option, +pub struct Bag { + head: Option>, + tail: Option>, + + #[codec(skip)] + bag_idx: BagIdx, } -pub type BagOf = Bag>; +impl Bag { + /// Get a bag by idx. + pub fn get(bag_idx: BagIdx) -> Option> { + crate::VoterBags::::try_get(bag_idx).ok().map(|mut bag| { + bag.bag_idx = bag_idx; + bag + }) + } + + /// Put the bag back into storage. + pub fn put(self) { + crate::VoterBags::::insert(self.bag_idx, self); + } +} /// A Node is the fundamental element comprising the singly-linked lists which for each bag. #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug))] -pub struct Node { - voter: Voter, - next: Option, +pub struct Node { + voter: Voter>, + next: Option>, + + #[codec(skip)] + bag_idx: BagIdx, } -pub type NodeOf = Node>; +impl Node { + /// Get a node by bag idx and account id. + pub fn get(bag_idx: BagIdx, account_id: &AccountIdOf) -> Option> { + crate::VoterNodes::::try_get(&bag_idx, account_id).ok().map(|mut node| { + node.bag_idx = bag_idx; + node + }) + } + + /// Get a node by account id. + /// + /// Note that this must perform two storage lookups: one to identify which bag is appropriate, + /// and another to actually fetch the node. + pub fn from_id(account_id: &AccountIdOf) -> Option> { + let bag = current_bag_for::(account_id)?; + Self::get(bag, account_id) + } + + /// Get a node by account id, assuming it's in the same bag as this node. + pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { + Self::get(self.bag_idx, account_id) + } + + /// Put the node back into storage. + pub fn put(self) { + crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); + } +} + +/// Fundamental information about a voter. +#[derive(Clone, Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Voter { + /// Account Id of this voter + pub id: AccountId, + /// Whether the voter is a validator or nominator + pub voter_type: VoterType, +} + +pub type VoterOf = Voter>; + +/// Type of voter. +/// +/// Similar to [`crate::StakerStatus`], but somewhat more limited. +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum VoterType { + Validator, + Nominator, +} From 450c3440c4df356c50dda55ef3dfad3d67ab6d6e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:27:23 +0200 Subject: [PATCH 03/82] node getters and iterators for VoterList, Bag --- frame/staking/src/voter_bags.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 8670197398ecf..39d469dee8055 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -60,6 +60,14 @@ impl VoterList { pub fn decode_len() -> Option { crate::VoterCount::try_get().ok().map(|n| n.saturated_into()) } + + /// Iterate over all nodes in all bags in the voter list. + /// + /// Note that this exhaustively attempts to try all possible bag indices. Full iteration can be + /// expensive; it's recommended to limit the number of items with `.take(n)`. + pub fn iter() -> impl Iterator> { + (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) + } } /// A Bag contains a singly-linked list of voters. @@ -91,6 +99,21 @@ impl Bag { pub fn put(self) { crate::VoterBags::::insert(self.bag_idx, self); } + + /// Get the head node in this bag. + pub fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + } + + /// Get the tail node in this bag. + pub fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + } + + /// Iterate over the nodes in this bag. + pub fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } } /// A Node is the fundamental element comprising the singly-linked lists which for each bag. @@ -131,6 +154,11 @@ impl Node { pub fn put(self) { crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); } + + /// Get the next node in the bag. + pub fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| self.in_bag(id)) + } } /// Fundamental information about a voter. From 21e79eece2afb6c37400f94bc2371082e8a465dd Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:37:11 +0200 Subject: [PATCH 04/82] simplify get_npos_voters --- frame/staking/src/lib.rs | 37 +++++++-------------------------- frame/staking/src/voter_bags.rs | 34 ++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 7dd8c19cd4487..0c7d7ce543f6c 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2521,39 +2521,18 @@ impl Module { /// auto-chilled. /// /// Note that this is VERY expensive. Use with care. - pub fn get_npos_voters( - maybe_max_len: Option, - ) -> Vec> { - let weight_of = Self::slashable_balance_of_fn(); - let mut all_voters = Vec::new(); - - for (validator, _) in >::iter() { - // append self vote - let self_vote = (validator.clone(), weight_of(&validator), vec![validator.clone()]); - all_voters.push(self_vote); - } + pub fn get_npos_voters(maybe_max_len: Option) -> Vec> { + let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); + let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); + let weight_of = Self::slashable_balance_of_fn(); // collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - for (nominator, nominations) in >::iter() { - let Nominations { submitted_in, mut targets, suppressed: _ } = nominations; - - // Filter out nomination targets which were nominated before the most recent - // slashing span. - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); - - if !targets.is_empty() { - let vote_weight = weight_of(&nominator); - all_voters.push((nominator, vote_weight, targets)) - } - } - - all_voters + voter_bags::VoterList::::iter() + .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) + .take(wanted_voters) + .collect() } pub fn get_npos_targets() -> Vec { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 39d469dee8055..39d34d4d3f749 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,12 +22,13 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Pallet, VoterBagFor, VotingDataOf, VoteWeight, + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, Pallet, VoterBagFor, + VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; use sp_runtime::SaturatedConversion; -use sp_std::marker::PhantomData; +use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; /// Index type for a bag. pub type BagIdx = u8; @@ -159,6 +160,35 @@ impl Node { pub fn next(&self) -> Option> { self.next.as_ref().and_then(|id| self.in_bag(id)) } + + /// Get this voter's voting data. + pub fn voting_data( + &self, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + slashing_spans: &BTreeMap, SlashingSpans>, + ) -> Option> { + match self.voter.voter_type { + VoterType::Validator => Some(( + self.voter.id.clone(), + weight_of(&self.voter.id), + vec![self.voter.id.clone()], + )), + VoterType::Nominator => { + let Nominations { submitted_in, mut targets, .. } = + Nominators::::get(self.voter.id.clone())?; + // Filter out nomination targets which were nominated before the most recent + // slashing span. + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + + (!targets.is_empty()) + .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) + } + } + } } /// Fundamental information about a voter. From f4998e2926e6ab1f40880c08bad2050e72137c07 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 15:43:39 +0200 Subject: [PATCH 05/82] simplify fn voters --- frame/staking/src/lib.rs | 24 +++++++++--------------- frame/staking/src/voter_bags.rs | 6 +++--- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 0c7d7ce543f6c..476e33602053f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2521,8 +2521,10 @@ impl Module { /// auto-chilled. /// /// Note that this is VERY expensive. Use with care. - pub fn get_npos_voters(maybe_max_len: Option) -> Vec> { - let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); + pub fn get_npos_voters( + maybe_max_len: Option, + voter_count: usize, + ) -> Vec> { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); let weight_of = Self::slashable_balance_of_fn(); @@ -2551,24 +2553,16 @@ impl frame_election_provider_support::ElectionDataProvider, ) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec)>, Weight)> { - // NOTE: reading these counts already needs to iterate a lot of storage keys, but they get - // cached. This is okay for the case of `Ok(_)`, but bad for `Err(_)`, as the trait does not - // report weight in failures. - let nominator_count = >::iter().count(); - let validator_count = >::iter().count(); - let voter_count = nominator_count.saturating_add(validator_count); - - if maybe_max_len.map_or(false, |max_len| voter_count > max_len) { - return Err("Voter snapshot too big"); - } + let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( - nominator_count as u32, - validator_count as u32, + // TODO: fix the weight calculation here + 0 as u32, + voter_count as u32, slashing_span_count as u32, ); - Ok((Self::get_npos_voters(maybe_max_len), weight)) + Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) } fn targets(maybe_max_len: Option) -> data_provider::Result<(Vec, Weight)> { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 39d34d4d3f749..f4a48de8a0afa 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,11 +22,11 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, Pallet, VoterBagFor, + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, VoterBagFor, VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; -use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; +use frame_support::{StorageMap, StorageValue, StorageDoubleMap}; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; @@ -34,7 +34,7 @@ use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; pub type BagIdx = u8; /// Given a certain vote weight, which bag should this voter contain? -fn notional_bag_for(weight: VoteWeight) -> BagIdx { +fn notional_bag_for(_weight: VoteWeight) -> BagIdx { todo!("geometric series of some description; ask alfonso") } From 8960daac0e6c54016f39adb1560058a09c8d3725 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 16:24:17 +0200 Subject: [PATCH 06/82] VoterList::insert --- frame/staking/src/voter_bags.rs | 61 +++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index f4a48de8a0afa..53dd68a0f5b14 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -26,7 +26,7 @@ use crate::{ VotingDataOf, VoteWeight, }; use codec::{Encode, Decode}; -use frame_support::{StorageMap, StorageValue, StorageDoubleMap}; +use frame_support::{DefaultNoBound, StorageMap, StorageValue, StorageDoubleMap}; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; @@ -69,6 +69,31 @@ impl VoterList { pub fn iter() -> impl Iterator> { (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) } + + /// Insert a new voter into the appropriate bag in the voter list. + pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { + Self::insert_many(sp_std::iter::once(voter), weight_of) + } + + /// Insert several voters into the appropriate bags in the voter list. + /// + /// This is more efficient than repeated calls to `Self::insert`. + pub fn insert_many( + voters: impl IntoIterator>, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + ) { + let mut bags = BTreeMap::new(); + + for voter in voters.into_iter() { + let weight = weight_of(&voter.id); + let bag = notional_bag_for(weight); + bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); + } + + for (_, bag) in bags { + bag.put(); + } + } } /// A Bag contains a singly-linked list of voters. @@ -78,7 +103,7 @@ impl VoterList { /// more desirable to ensure that there is some element of first-come, first-serve to the list's /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. -#[derive(Default, Encode, Decode)] +#[derive(DefaultNoBound, Encode, Decode)] pub struct Bag { head: Option>, tail: Option>, @@ -96,6 +121,11 @@ impl Bag { }) } + /// Get a bag by idx or make it, appropriately initialized. + pub fn get_or_make(bag_idx: BagIdx) -> Bag { + Self::get(bag_idx).unwrap_or(Bag { bag_idx, ..Default::default() }) + } + /// Put the bag back into storage. pub fn put(self) { crate::VoterBags::::insert(self.bag_idx, self); @@ -115,6 +145,33 @@ impl Bag { pub fn iter(&self) -> impl Iterator> { sp_std::iter::successors(self.head(), |prev| prev.next()) } + + /// Insert a new voter into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + fn insert(&mut self, voter: VoterOf) { + let id = voter.id.clone(); + + // insert the actual voter + let voter_node = Node:: { voter, next: None, bag_idx: self.bag_idx }; + voter_node.put(); + + // update the previous tail + if let Some(mut tail) = self.tail() { + tail.next = Some(id.clone()); + tail.put(); + } + + // update the internal bag links + if self.head.is_none() { + self.head = Some(id.clone()); + } + self.tail = Some(id); + } } /// A Node is the fundamental element comprising the singly-linked lists which for each bag. From f9d061d55f896b0a7f185d930150f869a98dda0e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 11 Jun 2021 16:51:29 +0200 Subject: [PATCH 07/82] VoterList::remove --- frame/staking/src/voter_bags.rs | 63 +++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 53dd68a0f5b14..556c7893b9a13 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -94,9 +94,54 @@ impl VoterList { bag.put(); } } + + /// Remove a voter (by id) from the voter list. + pub fn remove(voter: &AccountIdOf) { + Self::remove_many(sp_std::iter::once(voter)) + } + + /// Remove many voters (by id) from the voter list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + pub fn remove_many<'a>(voters: impl IntoIterator>) { + let mut bags = BTreeMap::new(); + + for voter_id in voters.into_iter() { + let node = match Node::::from_id(voter_id) { + Some(node) => node, + None => continue, + }; + + // modify the surrounding nodes + if let Some(mut prev) = node.prev() { + prev.next = node.next.clone(); + prev.put(); + } + if let Some(mut next) = node.next() { + next.prev = node.prev.clone(); + next.put(); + } + + // clear the bag head/tail pointers as necessary + let mut bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); + if bag.head.as_ref() == Some(voter_id) { + bag.head = node.next; + } + if bag.tail.as_ref() == Some(voter_id) { + bag.tail = node.prev; + } + + // now get rid of the node itself + crate::VoterNodes::::remove(node.bag_idx, voter_id); + } + + for (_, bag) in bags { + bag.put(); + } + } } -/// A Bag contains a singly-linked list of voters. +/// A Bag is a doubly-linked list of voters. /// /// Note that we maintain both head and tail pointers. While it would be possible to get away /// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's @@ -155,13 +200,19 @@ impl Bag { /// `self.put()` after use. fn insert(&mut self, voter: VoterOf) { let id = voter.id.clone(); + let tail = self.tail(); // insert the actual voter - let voter_node = Node:: { voter, next: None, bag_idx: self.bag_idx }; + let voter_node = Node:: { + voter, + prev: tail.as_ref().map(|prev| prev.voter.id.clone()), + next: None, + bag_idx: self.bag_idx, + }; voter_node.put(); // update the previous tail - if let Some(mut tail) = self.tail() { + if let Some(mut tail) = tail { tail.next = Some(id.clone()); tail.put(); } @@ -179,6 +230,7 @@ impl Bag { #[cfg_attr(feature = "std", derive(Debug))] pub struct Node { voter: Voter>, + prev: Option>, next: Option>, #[codec(skip)] @@ -213,6 +265,11 @@ impl Node { crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); } + /// Get the previous node in the bag. + pub fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| self.in_bag(id)) + } + /// Get the next node in the bag. pub fn next(&self) -> Option> { self.next.as_ref().and_then(|id| self.in_bag(id)) From 1e7dfd96a898eb2e01e430625e0cc635f1539682 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 14 Jun 2021 12:58:30 +0200 Subject: [PATCH 08/82] nodes are doubly-linked --- frame/staking/src/voter_bags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 556c7893b9a13..77becfb5f78c6 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -225,7 +225,7 @@ impl Bag { } } -/// A Node is the fundamental element comprising the singly-linked lists which for each bag. +/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(Debug))] pub struct Node { From c6bcc760bf4fcad12d8f06225532b9262b98c345 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 14 Jun 2021 16:53:16 +0200 Subject: [PATCH 09/82] impl fn update_position_for, update voter counts appropriately --- frame/staking/src/voter_bags.rs | 125 +++++++++++++++++++++++++------- 1 file changed, 99 insertions(+), 26 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index b430494677168..fb44797bcd2ef 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -58,6 +58,7 @@ fn current_bag_for(id: &AccountIdOf) -> Option { pub struct VoterList(PhantomData); impl VoterList { + /// Decode the length of the voter list. pub fn decode_len() -> Option { crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) } @@ -83,16 +84,20 @@ impl VoterList { weight_of: impl Fn(&T::AccountId) -> VoteWeight, ) { let mut bags = BTreeMap::new(); + let mut count = 0; for voter in voters.into_iter() { let weight = weight_of(&voter.id); let bag = notional_bag_for(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); + count += 1; } for (_, bag) in bags { bag.put(); } + + crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); } /// Remove a voter (by id) from the voter list. @@ -105,31 +110,18 @@ impl VoterList { /// This is more efficient than repeated calls to `Self::remove`. pub fn remove_many<'a>(voters: impl IntoIterator>) { let mut bags = BTreeMap::new(); + let mut count = 0; for voter_id in voters.into_iter() { let node = match Node::::from_id(voter_id) { Some(node) => node, None => continue, }; - - // modify the surrounding nodes - if let Some(mut prev) = node.prev() { - prev.next = node.next.clone(); - prev.put(); - } - if let Some(mut next) = node.next() { - next.prev = node.prev.clone(); - next.put(); - } + count += 1; // clear the bag head/tail pointers as necessary - let mut bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); - if bag.head.as_ref() == Some(voter_id) { - bag.head = node.next; - } - if bag.tail.as_ref() == Some(voter_id) { - bag.tail = node.prev; - } + let bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); + bag.remove_node(&node); // now get rid of the node itself crate::VoterNodes::::remove(node.bag_idx, voter_id); @@ -138,6 +130,39 @@ impl VoterList { for (_, bag) in bags { bag.put(); } + + crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_sub(count)); + } + + /// Update a voter's position in the voter list. + /// + /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `true` if the voter moved. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub fn update_position_for( + mut node: Node, + weight_of: impl Fn(&AccountIdOf) -> VoteWeight, + ) -> bool { + let was_misplaced = node.is_misplaced(&weight_of); + if was_misplaced { + // clear the old bag head/tail pointers as necessary + if let Some(mut bag) = Bag::::get(node.bag_idx) { + bag.remove_node(&node); + bag.put(); + } + + // put the voter into the appropriate new bag + node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); + let mut bag = Bag::::get_or_make(node.bag_idx); + bag.insert_node(node); + bag.put(); + } + was_misplaced } } @@ -199,20 +224,30 @@ impl Bag { /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. fn insert(&mut self, voter: VoterOf) { - let id = voter.id.clone(); - let tail = self.tail(); - - // insert the actual voter - let voter_node = Node:: { + self.insert_node(Node:: { voter, - prev: tail.as_ref().map(|prev| prev.voter.id.clone()), + prev: None, next: None, bag_idx: self.bag_idx, - }; - voter_node.put(); + }); + } + + /// Insert a voter node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + fn insert_node(&mut self, mut node: Node) { + let id = node.voter.id.clone(); + + node.prev = self.tail.clone(); + node.next = None; + node.put(); // update the previous tail - if let Some(mut tail) = tail { + if let Some(mut tail) = self.tail() { tail.next = Some(id.clone()); tail.put(); } @@ -223,6 +258,25 @@ impl Bag { } self.tail = Some(id); } + + /// Remove a voter node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the voter in + /// the first place. Generally, use [`VoterList::remove`] instead. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()` and `node.put()` after use. + fn remove_node(&mut self, node: &Node) { + node.excise(); + + // clear the bag head/tail pointers as necessary + if self.head.as_ref() == Some(&node.voter.id) { + self.head = node.next.clone(); + } + if self.tail.as_ref() == Some(&node.voter.id) { + self.tail = node.prev.clone(); + } + } } /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. @@ -303,6 +357,25 @@ impl Node { } } } + + /// Remove this node from the linked list. + /// + /// Modifies storage, but only modifies the adjacent nodes. Does not modify `self` or any bag. + fn excise(&self) { + if let Some(mut prev) = self.prev() { + prev.next = self.next.clone(); + prev.put(); + } + if let Some(mut next) = self.next() { + next.prev = self.prev.clone(); + next.put(); + } + } + + /// `true` when this voter is in the wrong bag. + pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { + notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx + } } /// Fundamental information about a voter. From 82154dd8cd7543a1c9802b96db163426361f455d Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 14 Jun 2021 16:58:42 +0200 Subject: [PATCH 10/82] keep VoterBagFor updated --- frame/staking/src/voter_bags.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index fb44797bcd2ef..3a6495c49b745 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -125,6 +125,7 @@ impl VoterList { // now get rid of the node itself crate::VoterNodes::::remove(node.bag_idx, voter_id); + crate::VoterBagFor::::remove(voter_id); } for (_, bag) in bags { @@ -256,7 +257,9 @@ impl Bag { if self.head.is_none() { self.head = Some(id.clone()); } - self.tail = Some(id); + self.tail = Some(id.clone()); + + crate::VoterBagFor::::insert(id, self.bag_idx); } /// Remove a voter node from this bag. From 294cd99a0b82dd9a8ee6d7c182842721ca41f32e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 15 Jun 2021 13:53:42 +0200 Subject: [PATCH 11/82] manipulate VoterList everywhere which seems relevant --- frame/staking/src/lib.rs | 21 ++++++++++++++------- frame/staking/src/voter_bags.rs | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 8bc9cd20dd38a..a2d32aa3f23b8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -312,11 +312,9 @@ use sp_staking::{ SessionIndex, offence::{OnOffenceHandler, OffenceDetails, Offence, ReportOffence, OffenceError}, }; -use frame_system::{ - ensure_signed, ensure_root, pallet_prelude::*, - offchain::SendTransactionTypes, -}; +use frame_system::{ensure_signed, ensure_root, pallet_prelude::*, offchain::SendTransactionTypes}; use frame_election_provider_support::{ElectionProvider, VoteWeight, Supports, data_provider}; +use voter_bags::{VoterList, VoterType}; pub use weights::WeightInfo; pub use pallet::*; @@ -1646,6 +1644,7 @@ pub mod pallet { let stash = &ledger.stash; >::remove(stash); >::insert(stash, prefs); + VoterList::::insert_as(stash, VoterType::Validator); Ok(()) } @@ -1699,6 +1698,7 @@ pub mod pallet { >::remove(stash); >::insert(stash, &nominations); + VoterList::::insert_as(stash, VoterType::Nominator); Ok(()) } @@ -1724,6 +1724,7 @@ pub mod pallet { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; Self::chill_stash(&ledger.stash); + VoterList::::remove(&ledger.stash); Ok(()) } @@ -2308,6 +2309,7 @@ impl Pallet { fn chill_stash(stash: &AccountIdOf) { >::remove(stash); >::remove(stash); + VoterList::::remove(stash); } /// Actually make a payment to a staker. This uses the currency's reward function @@ -2627,6 +2629,8 @@ impl Pallet { >::remove(stash); >::remove(stash); + VoterList::::remove(stash); + frame_system::Pallet::::dec_consumers(stash); Ok(()) @@ -2725,7 +2729,7 @@ impl Pallet { // collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); - voter_bags::VoterList::::iter() + VoterList::::iter() .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) .take(wanted_voters) .collect() @@ -2736,7 +2740,8 @@ impl Pallet { } } -impl frame_election_provider_support::ElectionDataProvider, BlockNumberFor> +impl + frame_election_provider_support::ElectionDataProvider, BlockNumberFor> for Pallet { const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; @@ -2747,7 +2752,7 @@ impl frame_election_provider_support::ElectionDataProvider, ) -> data_provider::Result<(Vec<(AccountIdOf, VoteWeight, Vec>)>, Weight)> { - let voter_count = voter_bags::VoterList::::decode_len().unwrap_or_default(); + let voter_count = VoterList::::decode_len().unwrap_or_default(); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( @@ -2819,6 +2824,7 @@ impl frame_election_provider_support::ElectionDataProvider::insert_as(&v, VoterType::Validator); >::insert( v, ValidatorPrefs { commission: Perbill::zero(), blocked: false }, @@ -2840,6 +2846,7 @@ impl frame_election_provider_support::ElectionDataProvider::insert_as(&v, VoterType::Nominator); >::insert( v, Nominations { targets: t, submitted_in: 0, suppressed: false }, diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 3a6495c49b745..f37cff1c99e8c 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -22,8 +22,8 @@ //! voters doesn't particularly matter. use crate::{ - slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, VoterBagFor, - VotingDataOf, VoteWeight, + AccountIdOf, Config, Nominations, Nominators, Pallet, VoteWeight, VoterBagFor, VotingDataOf, + slashing::SlashingSpans, }; use codec::{Encode, Decode}; use frame_support::DefaultNoBound; @@ -71,6 +71,20 @@ impl VoterList { (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) } + /// Insert a new voter into the appropriate bag in the voter list. + /// + /// If the voter is already present in the list, their type will be updated. + /// That case is cheaper than inserting a new voter. + pub fn insert_as(account_id: &AccountIdOf, voter_type: VoterType) { + // if this is an update operation we can complete this easily and cheaply + if !Node::::update_voter_type_for(account_id, voter_type) { + // otherwise, we need to insert from scratch + let weight_of = Pallet::::slashable_balance_of_fn(); + let voter = Voter { id: account_id.clone(), voter_type }; + Self::insert(voter, weight_of); + } + } + /// Insert a new voter into the appropriate bag in the voter list. pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { Self::insert_many(sp_std::iter::once(voter), weight_of) @@ -379,6 +393,21 @@ impl Node { pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx } + + /// Update the voter type associated with a particular node by id. + /// + /// This updates storage immediately. + /// + /// Returns whether the voter existed and was successfully updated. + pub fn update_voter_type_for(account_id: &AccountIdOf, voter_type: VoterType) -> bool { + let node = Self::from_id(account_id); + let existed = node.is_some(); + if let Some(mut node) = node { + node.voter.voter_type = voter_type; + node.put(); + } + existed + } } /// Fundamental information about a voter. From 9787daf1372742473d6098696a3bfd7c20017975 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 15 Jun 2021 16:59:43 +0200 Subject: [PATCH 12/82] start sketching out `notional_bag_for` --- frame/staking/src/voter_bags.rs | 69 ++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index f37cff1c99e8c..80d30fc51249a 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -33,9 +33,68 @@ use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; /// Index type for a bag. pub type BagIdx = u8; +/// How many bags there are +const N_BAGS: BagIdx = 200; + /// Given a certain vote weight, which bag should this voter contain? -fn notional_bag_for(_weight: VoteWeight) -> BagIdx { - todo!("geometric series of some description; ask alfonso") +/// +/// Bags are separated by fixed thresholds. The formula for a threshold is, for `t` in `0..=N_BAGS`: +/// +/// 10 ^ ((t / 10) - 10) +/// +/// Given `N_BAGS == 200`, this means that the lowest threshold is `10^-10`, and the highest is +/// `10^10`. A given vote weight always fits into the bag `t` such that `threshold(t-1) <= weight < +/// threshold(t)`. We can determine an appropriate value for `t` by binary search. +/// +/// It is important for the correctness of the iteration algorithm that the bags of highest value +/// have the lowest threshold. Therefore, the appropriate `BagIdx` for a given value `T` is +/// `N_BAGS - t`. +fn notional_bag_for(weight: VoteWeight) -> BagIdx { + // the input to this threshold function is _not_ reversed; `threshold(0) == 0` + let threshold = |bag: BagIdx| -> u64 { + // The goal is to segment the full range of `u64` into `N_BAGS`, such that `threshold(0) != 0`, + // `threshold(N_BAGS) == u64::MAX`, and for all `t` in `0..N_BAGS`, + // `threshold(t + 1) as f64 / threshold(t) as f64 == CONSTANT_RATIO`. For `N_BAGS == 200`, + // `CONSTANT_RATIO ~= 1.25`. + // + // The natural, simple implementation here is + // + // ```rust + // // float exp and ln are not constant functions, unfortunately + // let CONSTANT_RATIO = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); + // CONSTANT_RATIO.powi(bag.into()).into() + // ``` + // + // Unfortunately, that doesn't quite work, for two reasons: + // + // - floats are nondeterministic and not allowed on the blockchain + // - f64 has insufficient capacity to completely and accurately compute f64 + // + // Perhaps the answer is going to end up being to bring in a fixed-point bignum implementation. + // See: https://docs.rs/fixed/1.9.0/fixed/struct.FixedU128.html + todo!() + }; + + // `want_bag` is the highest bag for which `threshold(want_bag) >= weight` + let want_bag = { + // TODO: use a binary search instead + let mut t = N_BAGS; + while t > 0 { + if threshold(t) >= weight { + break; + } + t -= 1; + } + t + }; + + debug_assert!( + (0..=N_BAGS).contains(&want_bag), + "must have computed a valid want_bag" + ); + + // reverse the index so that iteration works properly + N_BAGS - want_bag } /// Find the actual bag containing the current voter. @@ -102,7 +161,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); - let bag = notional_bag_for(weight); + let bag = notional_bag_for::(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -172,7 +231,7 @@ impl VoterList { } // put the voter into the appropriate new bag - node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); + node.bag_idx = notional_bag_for::(weight_of(&node.voter.id)); let mut bag = Bag::::get_or_make(node.bag_idx); bag.insert_node(node); bag.put(); @@ -391,7 +450,7 @@ impl Node { /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx + notional_bag_for::(weight_of(&self.voter.id)) != self.bag_idx } /// Update the voter type associated with a particular node by id. From a857f596348359761acff8ea4f3ba171ce787965 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 15 Jun 2021 17:01:11 +0200 Subject: [PATCH 13/82] `notional_bag_for` doesn't need a type parameter --- frame/staking/src/voter_bags.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 80d30fc51249a..009a943b644a8 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -49,7 +49,7 @@ const N_BAGS: BagIdx = 200; /// It is important for the correctness of the iteration algorithm that the bags of highest value /// have the lowest threshold. Therefore, the appropriate `BagIdx` for a given value `T` is /// `N_BAGS - t`. -fn notional_bag_for(weight: VoteWeight) -> BagIdx { +fn notional_bag_for(weight: VoteWeight) -> BagIdx { // the input to this threshold function is _not_ reversed; `threshold(0) == 0` let threshold = |bag: BagIdx| -> u64 { // The goal is to segment the full range of `u64` into `N_BAGS`, such that `threshold(0) != 0`, @@ -161,7 +161,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); - let bag = notional_bag_for::(weight); + let bag = notional_bag_for(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -231,7 +231,7 @@ impl VoterList { } // put the voter into the appropriate new bag - node.bag_idx = notional_bag_for::(weight_of(&node.voter.id)); + node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); let mut bag = Bag::::get_or_make(node.bag_idx); bag.insert_node(node); bag.put(); @@ -450,7 +450,7 @@ impl Node { /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for::(weight_of(&self.voter.id)) != self.bag_idx + notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx } /// Update the voter type associated with a particular node by id. From 1a50bbafa958bc8315c9e0dd9b89b2a08b071123 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 17 Jun 2021 17:04:32 +0200 Subject: [PATCH 14/82] implement voter bag thresholds in terms of a precomputed list --- frame/staking/Cargo.toml | 5 + frame/staking/src/bin/make_bags.rs | 40 +++ .../src/{voter_bags.rs => voter_bags/mod.rs} | 70 ++---- frame/staking/src/voter_bags/thresholds.rs | 229 ++++++++++++++++++ 4 files changed, 288 insertions(+), 56 deletions(-) create mode 100644 frame/staking/src/bin/make_bags.rs rename frame/staking/src/{voter_bags.rs => voter_bags/mod.rs} (86%) create mode 100644 frame/staking/src/voter_bags/thresholds.rs diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 908e361e667e3..ce0a394df4167 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -70,3 +70,8 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] +make-bags = [] + +[[bin]] +name = "make_bags" +required-features = ["make-bags", "std"] diff --git a/frame/staking/src/bin/make_bags.rs b/frame/staking/src/bin/make_bags.rs new file mode 100644 index 0000000000000..664c7b5470ba0 --- /dev/null +++ b/frame/staking/src/bin/make_bags.rs @@ -0,0 +1,40 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Make the set of voting bag thresholds to be used in `voter_bags.rs`. + +use pallet_staking::voter_bags::N_BAGS; + +fn main() { + let ratio = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); + println!("pub const CONSTANT_RATIO: f64 = {};", ratio); + + let mut thresholds = Vec::with_capacity(N_BAGS as usize); + + while thresholds.len() < N_BAGS as usize { + let prev_item: u64 = thresholds.last().copied().unwrap_or_default(); + let item = (prev_item as f64 * ratio).max(prev_item as f64 + 1.0); + thresholds.push(item as u64); + } + + *thresholds.last_mut().unwrap() = u64::MAX; + + println!("pub const THRESHOLDS: [u64; {}] = {:#?};", N_BAGS, thresholds); + + debug_assert_eq!(thresholds.len(), N_BAGS as usize); + debug_assert_eq!(*thresholds.last().unwrap(), u64::MAX); +} diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags/mod.rs similarity index 86% rename from frame/staking/src/voter_bags.rs rename to frame/staking/src/voter_bags/mod.rs index 009a943b644a8..cb8909c74475a 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -21,6 +21,9 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. +mod thresholds; + +use thresholds::THRESHOLDS; use crate::{ AccountIdOf, Config, Nominations, Nominators, Pallet, VoteWeight, VoterBagFor, VotingDataOf, slashing::SlashingSpans, @@ -34,67 +37,22 @@ use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; pub type BagIdx = u8; /// How many bags there are -const N_BAGS: BagIdx = 200; +pub const N_BAGS: BagIdx = 200; /// Given a certain vote weight, which bag should this voter contain? /// -/// Bags are separated by fixed thresholds. The formula for a threshold is, for `t` in `0..=N_BAGS`: -/// -/// 10 ^ ((t / 10) - 10) +/// Bags are separated by fixed thresholds. To the extent possible, each threshold is a constant +/// small multiple of the one before it. That ratio is [`thresholds::CONSTANT_RATIO`]. The exception +/// are the smallest bags, which are each at least 1 greater than the previous, and the largest bag, +/// which is defined as `u64::MAX`. /// -/// Given `N_BAGS == 200`, this means that the lowest threshold is `10^-10`, and the highest is -/// `10^10`. A given vote weight always fits into the bag `t` such that `threshold(t-1) <= weight < -/// threshold(t)`. We can determine an appropriate value for `t` by binary search. -/// -/// It is important for the correctness of the iteration algorithm that the bags of highest value -/// have the lowest threshold. Therefore, the appropriate `BagIdx` for a given value `T` is -/// `N_BAGS - t`. +/// Bags are arranged such that `bags[0]` is the largest bag, and `bags[N_BAGS-1]` is the smallest. fn notional_bag_for(weight: VoteWeight) -> BagIdx { - // the input to this threshold function is _not_ reversed; `threshold(0) == 0` - let threshold = |bag: BagIdx| -> u64 { - // The goal is to segment the full range of `u64` into `N_BAGS`, such that `threshold(0) != 0`, - // `threshold(N_BAGS) == u64::MAX`, and for all `t` in `0..N_BAGS`, - // `threshold(t + 1) as f64 / threshold(t) as f64 == CONSTANT_RATIO`. For `N_BAGS == 200`, - // `CONSTANT_RATIO ~= 1.25`. - // - // The natural, simple implementation here is - // - // ```rust - // // float exp and ln are not constant functions, unfortunately - // let CONSTANT_RATIO = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); - // CONSTANT_RATIO.powi(bag.into()).into() - // ``` - // - // Unfortunately, that doesn't quite work, for two reasons: - // - // - floats are nondeterministic and not allowed on the blockchain - // - f64 has insufficient capacity to completely and accurately compute f64 - // - // Perhaps the answer is going to end up being to bring in a fixed-point bignum implementation. - // See: https://docs.rs/fixed/1.9.0/fixed/struct.FixedU128.html - todo!() - }; - - // `want_bag` is the highest bag for which `threshold(want_bag) >= weight` - let want_bag = { - // TODO: use a binary search instead - let mut t = N_BAGS; - while t > 0 { - if threshold(t) >= weight { - break; - } - t -= 1; - } - t - }; - - debug_assert!( - (0..=N_BAGS).contains(&want_bag), - "must have computed a valid want_bag" - ); - - // reverse the index so that iteration works properly - N_BAGS - want_bag + let raw_bag = match THRESHOLDS.binary_search(&weight) { + Ok(bag) => bag, + Err(bag) => bag, + } as BagIdx; + N_BAGS - raw_bag } /// Find the actual bag containing the current voter. diff --git a/frame/staking/src/voter_bags/thresholds.rs b/frame/staking/src/voter_bags/thresholds.rs new file mode 100644 index 0000000000000..710403d75e64f --- /dev/null +++ b/frame/staking/src/voter_bags/thresholds.rs @@ -0,0 +1,229 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Generated voter bag thresholds. + +use super::N_BAGS; + +/// Ratio between adjacent bags; +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 1.2483305489016119; + +/// Upper thresholds for each bag. +pub const THRESHOLDS: [u64; N_BAGS as usize] = [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 11, + 13, + 16, + 19, + 23, + 28, + 34, + 42, + 52, + 64, + 79, + 98, + 122, + 152, + 189, + 235, + 293, + 365, + 455, + 567, + 707, + 882, + 1101, + 1374, + 1715, + 2140, + 2671, + 3334, + 4161, + 5194, + 6483, + 8092, + 10101, + 12609, + 15740, + 19648, + 24527, + 30617, + 38220, + 47711, + 59559, + 74349, + 92812, + 115860, + 144631, + 180547, + 225382, + 281351, + 351219, + 438437, + 547314, + 683228, + 852894, + 1064693, + 1329088, + 1659141, + 2071156, + 2585487, + 3227542, + 4029039, + 5029572, + 6278568, + 7837728, + 9784075, + 12213759, + 15246808, + 19033056, + 23759545, + 29659765, + 37025190, + 46219675, + 57697432, + 72025466, + 89911589, + 112239383, + 140111850, + 174905902, + 218340380, + 272560966, + 340246180, + 424739700, + 530215542, + 661884258, + 826250339, + 1031433539, + 1287569995, + 1607312958, + 2006457867, + 2504722650, + 3126721800, + 3903182340, + 4872461752, + 6082442853, + 7592899225, + 9478448057, + 11832236265, + 14770541991, + 18438518791, + 23017366283, + 28733281486, + 35868633049, + 44775910382, + 55895136784, + 69775606782, + 87103021514, + 108733362657, + 135735178289, + 169442369618, + 211520086272, + 264046985399, + 329617918218, + 411472116776, + 513653213392, + 641208997818, + 800440780206, + 999214678517, + 1247350208103, + 1557105369953, + 1943782201171, + 2426482702132, + 3029052483452, + 3781258749319, + 4720260810076, + 5892445768000, + 7355720059940, + 9182370059991, + 11462633057206, + 14309155016159, + 17862555335640, + 22298373506924, + 27835740839511, + 34748205641269, + 43377246621511, + 54149142084871, + 67596028261358, + 84382187063069, + 105336861893959, + 131495222627659, + 164149503440725, + 204912839732087, + 255798957699744, + 319321653273781, + 398618974707429, + 497608243499122, + 621179571745225, + 775437435763184, + 968002239825113, + 1208386767378873, + 1508466116607513, + 1883064335344139, + 2350686735357198, + 2934434062644189, + 3663143684136207, + 4572814165923224, + 5708383617772005, + 7125949654914296, + 8895540644164415, + 11104575135106362, + 13862180373726516, + 17304583234907174, + 21601839888145304, + 26966236644853160, + 33662776992680304, + 42022272880825152, + 52457686971413784, + 65484533171133904, + 81746343238087392, + 102046457525101200, + 127387710335774608, + 159021970366777056, + 198511983555374656, + 247808573395228608, + 309347012448991104, + 386167325851522816, + 482064469848099072, + 601775804251442048, + 751215120036911616, + 937764783138868096, + 1170640426476344320, + 1461346206149632000, + 1824243111658058240, + 2277258404906088192, + 2842771234587226112, + 3548718175673985024, + 4429973308136232448, + 5530071011365192704, + 6903356581082402816, + 8617670910126150656, + 10757701857491230720, + 13429167864681918464, + 18446744073709551615, +]; From 721c2cb5c665b0d6f9df91b675b876868d757bad Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 18 Jun 2021 12:27:39 +0200 Subject: [PATCH 15/82] fix a failing test --- Cargo.lock | 1 + .../election-provider-support/src/onchain.rs | 7 +--- frame/staking/Cargo.toml | 1 + frame/staking/src/tests.rs | 9 ++-- primitives/npos-elections/src/lib.rs | 42 +++++++++++++++---- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb944b782abd9..034414e675fd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5501,6 +5501,7 @@ dependencies = [ "sp-application-crypto", "sp-core", "sp-io", + "sp-npos-elections", "sp-runtime", "sp-staking", "sp-std", diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index e034a9c36a8ac..89358fce12af8 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -77,11 +77,8 @@ impl ElectionProvider for OnChainSequen let (desired_targets, _) = Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; - let mut stake_map: BTreeMap = BTreeMap::new(); - - voters.iter().for_each(|(v, s, _)| { - stake_map.insert(v.clone(), *s); - }); + let stake_map: BTreeMap = + voters.iter().map(|(v, s, _)| (v.clone(), *s)).collect(); let stake_of = |w: &T::AccountId| -> VoteWeight { stake_map.get(w).cloned().unwrap_or_default() diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index ce0a394df4167..3e080f2537014 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -37,6 +37,7 @@ rand_chacha = { version = "0.2", default-features = false, optional = true } sp-storage = { version = "3.0.0", path = "../../primitives/storage" } sp-tracing = { version = "3.0.0", path = "../../primitives/tracing" } sp-core = { version = "3.0.0", path = "../../primitives/core" } +sp-npos-elections = { version = "3.0.0", path = "../../primitives/npos-elections", features = ["mocks"] } pallet-balances = { version = "3.0.0", path = "../balances" } pallet-timestamp = { version = "3.0.0", path = "../timestamp" } pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 5d42d866b1336..e7a7fe1178387 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -19,6 +19,7 @@ use super::{*, Event}; use mock::*; +use sp_npos_elections::supports_eq_unordered; use sp_runtime::{ assert_eq_error_rate, traits::{BadOrigin, Dispatchable}, @@ -1905,13 +1906,13 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider_elected() { // winners should be 21 and 11. let supports = ::ElectionProvider::elect().unwrap().0; - assert_eq!( - supports, - vec![ + assert!(supports_eq_unordered( + &supports, + &vec![ (11, Support { total: 1500, voters: vec![(11, 1000), (1, 500)] }), (21, Support { total: 2500, voters: vec![(21, 1000), (3, 1000), (1, 500)] }) ], - ); + )); }); } diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index c1cf41a40f2b5..5563b75e69eff 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -476,6 +476,18 @@ pub struct Support { pub voters: Vec<(AccountId, ExtendedBalance)>, } +#[cfg(feature = "mocks")] +impl Support { + /// `true` when the support is identical except for the ordering of the voters. + pub fn eq_unordered(&self, other: &Self) -> bool { + self.total == other.total && { + let my_voters: BTreeMap<_, _> = self.voters.iter().cloned().collect(); + let other_voters: BTreeMap<_, _> = other.voters.iter().cloned().collect(); + my_voters == other_voters + } + } +} + /// A target-major representation of the the election outcome. /// /// Essentially a flat variant of [`SupportMap`]. @@ -483,6 +495,18 @@ pub struct Support { /// The main advantage of this is that it is encodable. pub type Supports = Vec<(A, Support)>; +#[cfg(feature = "mocks")] +pub fn supports_eq_unordered(a: &Supports, b: &Supports) -> bool { + let map: BTreeMap<_, _> = a.iter().cloned().collect(); + b.iter().all(|(id, b_support)| { + let a_support = match map.get(id) { + Some(support) => support, + None => return false, + }; + a_support.eq_unordered(b_support) + }) +} + /// Linkage from a winner to their [`Support`]. /// /// This is more helpful than a normal [`Supports`] as it allows faster error checking. @@ -504,12 +528,12 @@ impl FlattenSupportMap for SupportMap { /// /// The list of winners is basically a redundancy for error checking only; It ensures that all the /// targets pointed to by the [`Assignment`] are present in the `winners`. -pub fn to_support_map( - winners: &[A], - assignments: &[StakedAssignment], -) -> Result, Error> { +pub fn to_support_map( + winners: &[AccountId], + assignments: &[StakedAssignment], +) -> Result, Error> { // Initialize the support of each candidate. - let mut supports = >::new(); + let mut supports = >::new(); winners.iter().for_each(|e| { supports.insert(e.clone(), Default::default()); }); @@ -532,10 +556,10 @@ pub fn to_support_map( /// flat vector. /// /// Similar to [`to_support_map`], `winners` is used for error checking. -pub fn to_supports( - winners: &[A], - assignments: &[StakedAssignment], -) -> Result, Error> { +pub fn to_supports( + winners: &[AccountId], + assignments: &[StakedAssignment], +) -> Result, Error> { to_support_map(winners, assignments).map(FlattenSupportMap::flatten) } From 92220ec6213f609dee9c528f501e8d7f73e4e4bd Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 18 Jun 2021 13:29:50 +0200 Subject: [PATCH 16/82] fix the rest of the non-benchmark tests --- frame/staking/src/tests.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index e7a7fe1178387..767192bd131ce 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -514,8 +514,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 800, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 400 }, IndividualExposure { who: 1, value: 400 }, + IndividualExposure { who: 3, value: 400 }, ] }, ); @@ -525,8 +525,8 @@ fn nominating_and_rewards_should_work() { total: 1000 + 1200, own: 1000, others: vec![ - IndividualExposure { who: 3, value: 600 }, IndividualExposure { who: 1, value: 600 }, + IndividualExposure { who: 3, value: 600 }, ] }, ); @@ -1860,13 +1860,13 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider() { // winners should be 21 and 31. Otherwise this election is taking duplicates into // account. let supports = ::ElectionProvider::elect().unwrap().0; - assert_eq!( - supports, - vec![ + assert!(supports_eq_unordered( + &supports, + &vec![ (21, Support { total: 1800, voters: vec![(21, 1000), (3, 400), (1, 400)] }), (31, Support { total: 2200, voters: vec![(31, 1000), (3, 600), (1, 600)] }) ], - ); + )); }); } @@ -3958,7 +3958,7 @@ mod election_data_provider { #[test] fn respects_len_limits() { ExtBuilder::default().build_and_execute(|| { - assert_eq!(Staking::voters(Some(1)).unwrap_err(), "Voter snapshot too big"); + assert_eq!(Staking::voters(Some(1)).unwrap().0.len(), 1); assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); }); } From 033d87faec30339328f87d2ec7abeb67bb3a3e42 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 10:14:31 +0200 Subject: [PATCH 17/82] revert an accidental change --- frame/staking/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index fed1e2df6598f..a1b3e948303cc 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2987,8 +2987,9 @@ impl Pallet { .collect() } - pub fn get_npos_targets() -> Vec> { - >::iter().map(|(v, _)| v).collect::>() + /// This is a very expensive function and result should be cached versus being called multiple times. + pub fn get_npos_targets() -> Vec { + Validators::::iter().map(|(v, _)| v).collect::>() } /// This function will add a nominator to the `Nominators` storage map, From 18c8eefdaaa75a52e9580032ee77eba0872fc5ec Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 16:43:44 +0200 Subject: [PATCH 18/82] adapt tests from benchmarking.rs for easier debugging --- frame/staking/src/tests.rs | 95 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 767192bd131ce..9746b059df105 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -17,6 +17,8 @@ //! Tests for the module. +use crate::voter_bags::Node; + use super::{*, Event}; use mock::*; use sp_npos_elections::supports_eq_unordered; @@ -30,6 +32,7 @@ use frame_support::{ traits::{Currency, ReservableCurrency, OnInitialize}, weights::{extract_actual_weight, GetDispatchInfo}, }; +use frame_system::RawOrigin; use pallet_balances::Error as BalancesError; use substrate_test_utils::assert_eq_uvec; use frame_election_provider_support::Support; @@ -3860,6 +3863,98 @@ fn on_finalize_weight_is_nonzero() { }) } + + +/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. +#[test] +fn payout_stakers_dead_controller() { + ExtBuilder::default() + .build_and_execute(|| { + // setup + let n = ::MaxNominatorRewardedPerValidator::get() as u32; + let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( + n, + ::MaxNominatorRewardedPerValidator::get() as u32, + true, + RewardDestination::Controller, + ).unwrap(); + + let current_era = CurrentEra::::get().unwrap(); + // set the commission for this particular era as well. + >::insert(current_era, validator.clone(), >::validators(&validator)); + + let caller = crate::benchmarking::whitelisted_caller(); + let validator_controller = >::get(&validator).unwrap(); + let balance_before = ::Currency::free_balance(&validator_controller); + for (_, controller) in &nominators { + let balance = ::Currency::free_balance(controller); + assert!(balance.is_zero(), "Controller has balance, but should be dead."); + } + + // benchmark + Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); + + // verify + let balance_after = ::Currency::free_balance(&validator_controller); + assert!( + balance_before < balance_after, + "Balance of validator controller should have increased after payout.", + ); + for (_, controller) in &nominators { + let balance = ::Currency::free_balance(controller); + if balance.is_zero() { + dbg!(controller, Node::::from_id(controller)); + } + assert!(!balance.is_zero(), "Payout not given to controller."); + } + }); +} + +/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. +#[test] +fn payout_stakers_alive_staked() { + ExtBuilder::default() + .build_and_execute(|| { + // setup + let n = ::MaxNominatorRewardedPerValidator::get() as u32; + let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( + n, + ::MaxNominatorRewardedPerValidator::get() as u32, + false, + RewardDestination::Staked, + ).unwrap(); + + let current_era = CurrentEra::::get().unwrap(); + // set the commission for this particular era as well. + >::insert(current_era, validator.clone(), >::validators(&validator)); + + let caller = crate::benchmarking::whitelisted_caller(); + let balance_before = ::Currency::free_balance(validator); + let mut nominator_balances_before = Vec::new(); + for (stash, _) in &nominators { + let balance = ::Currency::free_balance(stash); + nominator_balances_before.push(balance); + } + + // benchmark + Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); + + // verify + let balance_after = ::Currency::free_balance(validator); + assert!( + balance_before < balance_after, + "Balance of validator stash should have increased after payout.", + ); + for ((stash, _), balance_before) in nominators.iter().zip(nominator_balances_before.iter()) { + let balance_after = ::Currency::free_balance(stash); + assert!( + balance_before < &balance_after, + "Balance of nominator stash should have increased after payout.", + ); + } + }); +} + mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; From 9bd6682231e263a821399793deac59c12599c87c Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 16:43:50 +0200 Subject: [PATCH 19/82] investigate a theory as to why the payout_stakers tests are failing The thought was that we shouldn't be tracking the stash ID as the VoterList::Nominator; instead, we should track its controller. Unfortunately, that theory was wrong, and this just makes things worse. --- frame/staking/src/benchmarking.rs | 1 + frame/staking/src/lib.rs | 43 +++++++++++++++++++----------- frame/staking/src/slashing.rs | 4 +-- frame/staking/src/testing_utils.rs | 1 + 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 189158cc5643b..1633544861752 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -85,6 +85,7 @@ pub fn create_validator_with_nominators( }; if i < n { Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), vec![stash_lookup.clone()])?; + assert!(crate::voter_bags::Node::::from_id(&n_controller).is_some()); nominators.push((n_stash, n_controller)); } } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index a1b3e948303cc..651bdf597530a 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1761,7 +1761,7 @@ pub mod pallet { ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); let stash = &ledger.stash; - Self::do_remove_nominator(stash); + Self::do_remove_nominator(stash, &controller); Self::do_add_validator(stash, prefs); Ok(()) } @@ -1824,7 +1824,7 @@ pub mod pallet { }; Self::do_remove_validator(stash); - Self::do_add_nominator(stash, nominations); + Self::do_add_nominator(stash, &controller, nominations); Ok(()) } @@ -1849,8 +1849,7 @@ pub mod pallet { pub fn chill(origin: OriginFor) -> DispatchResult { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - Self::chill_stash(&ledger.stash); - VoterList::::remove(&ledger.stash); + Self::chill_stash(&ledger.stash, controller.into()); Ok(()) } @@ -2361,7 +2360,7 @@ pub mod pallet { ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); } - Self::chill_stash(&stash); + Self::chill_stash(&stash, controller.into()); Ok(()) } } @@ -2524,9 +2523,15 @@ impl Pallet { } /// Chill a stash account. - fn chill_stash(stash: &T::AccountId) { + /// + /// If `controller.is_none()`, looks it up in the ledger. However, if the controller is avilable + /// in the calling scope, it's more efficient to pass it directly. + fn chill_stash(stash: &T::AccountId, controller: Option) { Self::do_remove_validator(stash); - Self::do_remove_nominator(stash); + let controller = controller.unwrap_or_else(|| { + Bonded::::get(stash).expect("TODO: fix this error handling") + }); + Self::do_remove_nominator(stash, &controller); } /// Actually make a payment to a staker. This uses the currency's reward function @@ -2875,7 +2880,7 @@ impl Pallet { >::remove(stash); Self::do_remove_validator(stash); - Self::do_remove_nominator(stash); + Self::do_remove_nominator(stash, &controller); VoterList::::remove(stash); @@ -2996,22 +3001,26 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. - pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { - if !Nominators::::contains_key(who) { + pub fn do_add_nominator( + stash: &T::AccountId, + controller: &T::AccountId, + nominations: Nominations, + ) { + if !Nominators::::contains_key(stash) { CounterForNominators::::mutate(|x| x.saturating_inc()) } - Nominators::::insert(who, nominations); - VoterList::::insert_as(who, VoterType::Nominator); + Nominators::::insert(stash, nominations); + VoterList::::insert_as(controller, VoterType::Nominator); } /// This function will remove a nominator from the `Nominators` storage map, /// and keep track of the `CounterForNominators`. - pub fn do_remove_nominator(who: &T::AccountId) { - if Nominators::::contains_key(who) { - Nominators::::remove(who); + pub fn do_remove_nominator(stash: &T::AccountId, controller: &T::AccountId) { + if Nominators::::contains_key(stash) { + Nominators::::remove(stash); CounterForNominators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); } + VoterList::::remove(controller); } /// This function will add a validator to the `Validators` storage map, @@ -3151,8 +3160,10 @@ impl claimed_rewards: vec![], }, ); + let controller = Bonded::::get(&v).unwrap(); Self::do_add_nominator( &v, + &controller, Nominations { targets: t, submitted_in: 0, suppressed: false }, ); }); diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index 1e959e9341add..ef2ad15aa724f 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -285,7 +285,7 @@ pub(crate) fn compute_slash(params: SlashParams) // chill the validator - it misbehaved in the current span and should // not continue in the next election. also end the slashing span. spans.end_span(now); - >::chill_stash(stash); + >::chill_stash(stash, None); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(stash).unwrap_or(false) { @@ -325,7 +325,7 @@ fn kick_out_if_recent( if spans.era_span(params.slash_era).map(|s| s.index) == Some(spans.span_index()) { spans.end_span(params.now); - >::chill_stash(params.stash); + >::chill_stash(params.stash, None); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(params.stash).unwrap_or(false) { diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index c643cb283373b..f85ae93e2416f 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -33,6 +33,7 @@ pub fn clear_validators_and_nominators() { CounterForValidators::::kill(); Nominators::::remove_all(None); CounterForNominators::::kill(); + VoterBags::::drain(); } /// Grab a funded user. From 7a74fcf0270563e2a54ddcbc47dc384484765c4e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 21 Jun 2021 16:45:52 +0200 Subject: [PATCH 20/82] Revert "investigate a theory as to why the payout_stakers tests are failing" This reverts commit 9bd6682231e263a821399793deac59c12599c87c. --- frame/staking/src/benchmarking.rs | 1 - frame/staking/src/lib.rs | 43 +++++++++++------------------- frame/staking/src/slashing.rs | 4 +-- frame/staking/src/testing_utils.rs | 1 - 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 1633544861752..189158cc5643b 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -85,7 +85,6 @@ pub fn create_validator_with_nominators( }; if i < n { Staking::::nominate(RawOrigin::Signed(n_controller.clone()).into(), vec![stash_lookup.clone()])?; - assert!(crate::voter_bags::Node::::from_id(&n_controller).is_some()); nominators.push((n_stash, n_controller)); } } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 651bdf597530a..a1b3e948303cc 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1761,7 +1761,7 @@ pub mod pallet { ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); let stash = &ledger.stash; - Self::do_remove_nominator(stash, &controller); + Self::do_remove_nominator(stash); Self::do_add_validator(stash, prefs); Ok(()) } @@ -1824,7 +1824,7 @@ pub mod pallet { }; Self::do_remove_validator(stash); - Self::do_add_nominator(stash, &controller, nominations); + Self::do_add_nominator(stash, nominations); Ok(()) } @@ -1849,7 +1849,8 @@ pub mod pallet { pub fn chill(origin: OriginFor) -> DispatchResult { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - Self::chill_stash(&ledger.stash, controller.into()); + Self::chill_stash(&ledger.stash); + VoterList::::remove(&ledger.stash); Ok(()) } @@ -2360,7 +2361,7 @@ pub mod pallet { ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); } - Self::chill_stash(&stash, controller.into()); + Self::chill_stash(&stash); Ok(()) } } @@ -2523,15 +2524,9 @@ impl Pallet { } /// Chill a stash account. - /// - /// If `controller.is_none()`, looks it up in the ledger. However, if the controller is avilable - /// in the calling scope, it's more efficient to pass it directly. - fn chill_stash(stash: &T::AccountId, controller: Option) { + fn chill_stash(stash: &T::AccountId) { Self::do_remove_validator(stash); - let controller = controller.unwrap_or_else(|| { - Bonded::::get(stash).expect("TODO: fix this error handling") - }); - Self::do_remove_nominator(stash, &controller); + Self::do_remove_nominator(stash); } /// Actually make a payment to a staker. This uses the currency's reward function @@ -2880,7 +2875,7 @@ impl Pallet { >::remove(stash); Self::do_remove_validator(stash); - Self::do_remove_nominator(stash, &controller); + Self::do_remove_nominator(stash); VoterList::::remove(stash); @@ -3001,26 +2996,22 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. - pub fn do_add_nominator( - stash: &T::AccountId, - controller: &T::AccountId, - nominations: Nominations, - ) { - if !Nominators::::contains_key(stash) { + pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { + if !Nominators::::contains_key(who) { CounterForNominators::::mutate(|x| x.saturating_inc()) } - Nominators::::insert(stash, nominations); - VoterList::::insert_as(controller, VoterType::Nominator); + Nominators::::insert(who, nominations); + VoterList::::insert_as(who, VoterType::Nominator); } /// This function will remove a nominator from the `Nominators` storage map, /// and keep track of the `CounterForNominators`. - pub fn do_remove_nominator(stash: &T::AccountId, controller: &T::AccountId) { - if Nominators::::contains_key(stash) { - Nominators::::remove(stash); + pub fn do_remove_nominator(who: &T::AccountId) { + if Nominators::::contains_key(who) { + Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); + VoterList::::remove(who); } - VoterList::::remove(controller); } /// This function will add a validator to the `Validators` storage map, @@ -3160,10 +3151,8 @@ impl claimed_rewards: vec![], }, ); - let controller = Bonded::::get(&v).unwrap(); Self::do_add_nominator( &v, - &controller, Nominations { targets: t, submitted_in: 0, suppressed: false }, ); }); diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index ef2ad15aa724f..1e959e9341add 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -285,7 +285,7 @@ pub(crate) fn compute_slash(params: SlashParams) // chill the validator - it misbehaved in the current span and should // not continue in the next election. also end the slashing span. spans.end_span(now); - >::chill_stash(stash, None); + >::chill_stash(stash); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(stash).unwrap_or(false) { @@ -325,7 +325,7 @@ fn kick_out_if_recent( if spans.era_span(params.slash_era).map(|s| s.index) == Some(spans.span_index()) { spans.end_span(params.now); - >::chill_stash(params.stash, None); + >::chill_stash(params.stash); // make sure to disable validator till the end of this session if T::SessionInterface::disable_validator(params.stash).unwrap_or(false) { diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index f85ae93e2416f..c643cb283373b 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -33,7 +33,6 @@ pub fn clear_validators_and_nominators() { CounterForValidators::::kill(); Nominators::::remove_all(None); CounterForNominators::::kill(); - VoterBags::::drain(); } /// Grab a funded user. From e0ae1c05651f5f05df27074e06d4cf3163f8940d Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 14:09:26 +0200 Subject: [PATCH 21/82] find threshold more directly This method is known to return values in the range `(0..N_BAGS)`; see https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=b49ec461d667184c6856f8da6a059f85 for demonstration. --- frame/staking/src/voter_bags/mod.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index cb8909c74475a..8046db211f2d2 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -48,11 +48,9 @@ pub const N_BAGS: BagIdx = 200; /// /// Bags are arranged such that `bags[0]` is the largest bag, and `bags[N_BAGS-1]` is the smallest. fn notional_bag_for(weight: VoteWeight) -> BagIdx { - let raw_bag = match THRESHOLDS.binary_search(&weight) { - Ok(bag) => bag, - Err(bag) => bag, - } as BagIdx; - N_BAGS - raw_bag + let raw_bag = + THRESHOLDS.partition_point(|&threshold| weight > threshold) as BagIdx; + N_BAGS - raw_bag - 1 } /// Find the actual bag containing the current voter. From 66996549360742657bc3f72a710aae0144f0d9c2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 14:48:59 +0200 Subject: [PATCH 22/82] fix failing benchmark tests --- frame/staking/src/benchmarking.rs | 8 +++ frame/staking/src/testing_utils.rs | 1 + frame/staking/src/tests.rs | 95 ----------------------------- frame/staking/src/voter_bags/mod.rs | 8 +++ 4 files changed, 17 insertions(+), 95 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 189158cc5643b..0e258491133e2 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -96,6 +96,14 @@ pub fn create_validator_with_nominators( assert_eq!(new_validators.len(), 1); assert_eq!(new_validators[0], v_stash, "Our validator was not selected!"); + assert_eq!( + VoterList::::decode_len().unwrap_or_default() as u32, + CounterForNominators::::get() + CounterForValidators::::get(), + "ensure storage has been mutated coherently", + ); + assert_ne!(CounterForValidators::::get(), 0); + assert_ne!(CounterForNominators::::get(), 0); + assert_ne!(VoterList::::decode_len().unwrap_or_default(), 0); // Give Era Points let reward = EraRewardPoints:: { diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index c643cb283373b..482fcad8fb73f 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -33,6 +33,7 @@ pub fn clear_validators_and_nominators() { CounterForValidators::::kill(); Nominators::::remove_all(None); CounterForNominators::::kill(); + VoterList::::clear(); } /// Grab a funded user. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 9746b059df105..767192bd131ce 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -17,8 +17,6 @@ //! Tests for the module. -use crate::voter_bags::Node; - use super::{*, Event}; use mock::*; use sp_npos_elections::supports_eq_unordered; @@ -32,7 +30,6 @@ use frame_support::{ traits::{Currency, ReservableCurrency, OnInitialize}, weights::{extract_actual_weight, GetDispatchInfo}, }; -use frame_system::RawOrigin; use pallet_balances::Error as BalancesError; use substrate_test_utils::assert_eq_uvec; use frame_election_provider_support::Support; @@ -3863,98 +3860,6 @@ fn on_finalize_weight_is_nonzero() { }) } - - -/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. -#[test] -fn payout_stakers_dead_controller() { - ExtBuilder::default() - .build_and_execute(|| { - // setup - let n = ::MaxNominatorRewardedPerValidator::get() as u32; - let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( - n, - ::MaxNominatorRewardedPerValidator::get() as u32, - true, - RewardDestination::Controller, - ).unwrap(); - - let current_era = CurrentEra::::get().unwrap(); - // set the commission for this particular era as well. - >::insert(current_era, validator.clone(), >::validators(&validator)); - - let caller = crate::benchmarking::whitelisted_caller(); - let validator_controller = >::get(&validator).unwrap(); - let balance_before = ::Currency::free_balance(&validator_controller); - for (_, controller) in &nominators { - let balance = ::Currency::free_balance(controller); - assert!(balance.is_zero(), "Controller has balance, but should be dead."); - } - - // benchmark - Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); - - // verify - let balance_after = ::Currency::free_balance(&validator_controller); - assert!( - balance_before < balance_after, - "Balance of validator controller should have increased after payout.", - ); - for (_, controller) in &nominators { - let balance = ::Currency::free_balance(controller); - if balance.is_zero() { - dbg!(controller, Node::::from_id(controller)); - } - assert!(!balance.is_zero(), "Payout not given to controller."); - } - }); -} - -/// adapted from benchmarking.rs for debugging purposes; it is failing for non-obvious reasons. -#[test] -fn payout_stakers_alive_staked() { - ExtBuilder::default() - .build_and_execute(|| { - // setup - let n = ::MaxNominatorRewardedPerValidator::get() as u32; - let (validator, nominators) = crate::benchmarking::create_validator_with_nominators::( - n, - ::MaxNominatorRewardedPerValidator::get() as u32, - false, - RewardDestination::Staked, - ).unwrap(); - - let current_era = CurrentEra::::get().unwrap(); - // set the commission for this particular era as well. - >::insert(current_era, validator.clone(), >::validators(&validator)); - - let caller = crate::benchmarking::whitelisted_caller(); - let balance_before = ::Currency::free_balance(validator); - let mut nominator_balances_before = Vec::new(); - for (stash, _) in &nominators { - let balance = ::Currency::free_balance(stash); - nominator_balances_before.push(balance); - } - - // benchmark - Staking::payout_stakers(RawOrigin::Signed(caller).into(), validator.clone(), current_era).unwrap(); - - // verify - let balance_after = ::Currency::free_balance(validator); - assert!( - balance_before < balance_after, - "Balance of validator stash should have increased after payout.", - ); - for ((stash, _), balance_before) in nominators.iter().zip(nominator_balances_before.iter()) { - let balance_after = ::Currency::free_balance(stash); - assert!( - balance_before < &balance_after, - "Balance of nominator stash should have increased after payout.", - ); - } - }); -} - mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 8046db211f2d2..625497a161ea8 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -73,6 +73,14 @@ fn current_bag_for(id: &AccountIdOf) -> Option { pub struct VoterList(PhantomData); impl VoterList { + /// Remove all data associated with the voter list from storage. + pub fn clear() { + crate::VoterCount::::kill(); + crate::VoterBagFor::::remove_all(None); + crate::VoterBags::::remove_all(None); + crate::VoterNodes::::remove_all(None); + } + /// Decode the length of the voter list. pub fn decode_len() -> Option { crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) From 871b5250ce80c3b8c2b125a20f14aa358e1bd561 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 15:04:05 +0200 Subject: [PATCH 23/82] debug_assert that voter count is accurate --- frame/staking/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index a1b3e948303cc..8c90b9db08dca 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2975,6 +2975,12 @@ impl Pallet { maybe_max_len: Option, voter_count: usize, ) -> Vec> { + debug_assert_eq!( + voter_count, + VoterList::::decode_len().unwrap_or_default(), + "voter_count must be accurate", + ); + let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); let weight_of = Self::slashable_balance_of_fn(); From f2214040dc74dbde1008e2087b9c50a78f2d534a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 15:56:26 +0200 Subject: [PATCH 24/82] add extrinsic to rebag a stash --- frame/staking/src/lib.rs | 34 ++++++++++++++++++++++++++++- frame/staking/src/voter_bags/mod.rs | 22 ++++++++++++++----- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 8c90b9db08dca..7318ee28c95a3 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -314,7 +314,7 @@ use sp_staking::{ }; use frame_system::{ensure_signed, ensure_root, pallet_prelude::*, offchain::SendTransactionTypes}; use frame_election_provider_support::{ElectionProvider, VoteWeight, Supports, data_provider}; -use voter_bags::{VoterList, VoterType}; +use voter_bags::{BagIdx, VoterList, VoterType}; pub use weights::WeightInfo; pub use pallet::*; @@ -1360,6 +1360,10 @@ pub mod pallet { Kicked(T::AccountId, T::AccountId), /// The election failed. No new era is planned. StakingElectionFailed, + /// Attempted to rebag an account. + /// + /// If the second parameter is not `None`, it is the `(from, to)` tuple of bag indices. + Rebag(T::AccountId, Option<(BagIdx, BagIdx)>), } #[pallet::error] @@ -2364,6 +2368,34 @@ pub mod pallet { Self::chill_stash(&stash); Ok(()) } + + /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its + /// stake that it should properly fall into a different bag than its current position. + /// + /// This will adjust its position into the appropriate bag. This will affect its position + /// among the nominator/validator set once the snapshot is prepared for the election. + /// + /// Anyone can call this function about any stash. + // + // TODO: benchmark + #[pallet::weight(0)] + pub fn rebag( + origin: OriginFor, + stash: AccountIdOf, + ) -> DispatchResult { + ensure_signed(origin)?; + + let weight_of = Self::slashable_balance_of_fn(); + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let moved = voter_bags::Node::::from_id(&stash).and_then(|node| { + VoterList::update_position_for(node, weight_of) + }); + + Self::deposit_event(Event::::Rebag(stash, moved)); + + Ok(()) + } } } diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 625497a161ea8..f47ff63b8ba4c 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -185,22 +185,32 @@ impl VoterList { pub fn update_position_for( mut node: Node, weight_of: impl Fn(&AccountIdOf) -> VoteWeight, - ) -> bool { - let was_misplaced = node.is_misplaced(&weight_of); - if was_misplaced { + ) -> Option<(BagIdx, BagIdx)> { + node.is_misplaced(&weight_of).then(move || { + let old_idx = node.bag_idx; + // clear the old bag head/tail pointers as necessary if let Some(mut bag) = Bag::::get(node.bag_idx) { bag.remove_node(&node); bag.put(); + } else { + debug_assert!(false, "every node must have an extant bag associated with it"); + crate::log!( + error, + "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", + node.voter.id, + ); } // put the voter into the appropriate new bag - node.bag_idx = notional_bag_for(weight_of(&node.voter.id)); + let new_idx = notional_bag_for(weight_of(&node.voter.id)); + node.bag_idx = new_idx; let mut bag = Bag::::get_or_make(node.bag_idx); bag.insert_node(node); bag.put(); - } - was_misplaced + + (old_idx, new_idx) + }) } } From 976c41b7c47131ce73012431bde3f4a00aba75c9 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 24 Jun 2021 16:54:40 +0200 Subject: [PATCH 25/82] WIP: rebag benchmark --- frame/staking/src/benchmarking.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 0e258491133e2..2f2b5e8437d05 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -637,6 +637,15 @@ benchmarks! { verify { assert!(!Validators::::contains_key(controller)); } + + rebag { + let caller = whitelisted_caller(); + let (stash, _controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + + // TODO: figure out what's the worst case scenario for this call (lots of other users in + // VoterList?) and arrange for that to be the case + + }: _(RawOrigin::Signed(caller), stash.clone()) } #[cfg(test)] From 37e666ac6b956717120e02a8c91907a20b5d3d71 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 25 Jun 2021 12:02:56 +0200 Subject: [PATCH 26/82] WIP: rebag benchmark Note: this currently fails because the deposit into the target stash appears not to have changed its appropriate bag. --- frame/staking/src/benchmarking.rs | 62 +++++++++++++++++++++++++++-- frame/staking/src/lib.rs | 6 +-- frame/staking/src/voter_bags/mod.rs | 17 ++++++-- 3 files changed, 75 insertions(+), 10 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 2f2b5e8437d05..f9be1fb2f61bb 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -639,13 +639,67 @@ benchmarks! { } rebag { - let caller = whitelisted_caller(); - let (stash, _controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // The most expensive case for this call: + // + // - It doesn't matter where in the origin bag the stash lies; the number of reads and + // writes is constant. We can use the case that the stash is the only one in the origin + // bag, for simplicity. + // - The destination bag is not empty, because then we need to update the `next` pointer + // of the previous node in addition to the work we do otherwise. + + // Clean up any existing state. + clear_validators_and_nominators::(); + + use crate::voter_bags::{Bag, Node}; + + // stash and controller control the node account, which is a validator for simplicity of setup + let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + + let prefs = ValidatorPrefs::default(); + whitelist_account!(controller); + Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; - // TODO: figure out what's the worst case scenario for this call (lots of other users in - // VoterList?) and arrange for that to be the case + // create another validator with 3x the stake + create_validators::(1, 300)?; + // update the stash account's value/weight + T::Currency::deposit_into_existing(&stash, T::Currency::minimum_balance() * 200_u32.into())?; + + // verify preconditions + let weight_of = Staking::::weight_of_fn(); + let node = Node::::from_id(&stash).ok_or("node not found for stash")?; + assert!( + node.is_misplaced(&weight_of), + "rebagging only makes sense when a node is misplaced", + ); + assert_eq!( + { + let origin_bag = Bag::::get(node.bag_idx).ok_or("origin bag not found")?; + origin_bag.iter().count() + }, + 1, + "stash should be the only node in origin bag", + ); + assert_ne!( + { + let destination_bag = Bag::::get(node.proper_bag_for()).ok_or("destination bag not found")?; + destination_bag.iter().count() + }, + 0, + "destination bag should not be empty", + ); + drop(node); + + // caller will call rebag + let caller = whitelisted_caller(); + // ensure it's distinct from the other accounts + assert_ne!(caller, stash); + assert_ne!(caller, controller); }: _(RawOrigin::Signed(caller), stash.clone()) + verify { + let node = Node::::from_id(&stash).ok_or("node not found for stash")?; + assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); + } } #[cfg(test)] diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 7318ee28c95a3..9004651d01f28 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2385,10 +2385,10 @@ pub mod pallet { ) -> DispatchResult { ensure_signed(origin)?; - let weight_of = Self::slashable_balance_of_fn(); // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. let moved = voter_bags::Node::::from_id(&stash).and_then(|node| { + let weight_of = Self::weight_of_fn(); VoterList::update_position_for(node, weight_of) }); @@ -2418,7 +2418,7 @@ impl Pallet { /// /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is /// important to be only used while the total issuance is not changing. - pub fn slashable_balance_of_fn() -> Box) -> VoteWeight> { + pub fn weight_of_fn() -> Box) -> VoteWeight> { // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still // compile, while some types in mock fail to resolve. let issuance = T::Currency::total_issuance(); @@ -3015,7 +3015,7 @@ impl Pallet { let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); - let weight_of = Self::slashable_balance_of_fn(); + let weight_of = Self::weight_of_fn(); // collect all slashing spans into a BTreeMap for further queries. let slashing_spans = >::iter().collect::>(); diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index f47ff63b8ba4c..ca65495faafb5 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -50,7 +50,7 @@ pub const N_BAGS: BagIdx = 200; fn notional_bag_for(weight: VoteWeight) -> BagIdx { let raw_bag = THRESHOLDS.partition_point(|&threshold| weight > threshold) as BagIdx; - N_BAGS - raw_bag - 1 + N_BAGS - 1 - raw_bag } /// Find the actual bag containing the current voter. @@ -102,7 +102,7 @@ impl VoterList { // if this is an update operation we can complete this easily and cheaply if !Node::::update_voter_type_for(account_id, voter_type) { // otherwise, we need to insert from scratch - let weight_of = Pallet::::slashable_balance_of_fn(); + let weight_of = Pallet::::weight_of_fn(); let voter = Voter { id: account_id.clone(), voter_type }; Self::insert(voter, weight_of); } @@ -337,8 +337,9 @@ pub struct Node { prev: Option>, next: Option>, + /// The bag index is not stored in storage, but injected during all fetch operations. #[codec(skip)] - bag_idx: BagIdx, + pub(crate) bag_idx: BagIdx, } impl Node { @@ -441,6 +442,16 @@ impl Node { } existed } + + /// Get the index of the bag that this node _should_ be in, given its vote weight. + /// + /// This is a helper intended only for benchmarking and should not be used in production. + #[cfg(feature = "runtime-benchmarks")] + pub fn proper_bag_for(&self) -> BagIdx { + let weight_of = crate::Pallet::::weight_of_fn(); + let current_weight = weight_of(&self.voter.id); + notional_bag_for(current_weight) + } } /// Fundamental information about a voter. From b75b55be5d558ab74275fa700809e2eeb38a8e32 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 25 Jun 2021 16:13:53 +0200 Subject: [PATCH 27/82] make rebag test into a test case for easier debugging --- frame/staking/src/tests.rs | 81 +++++++++++++++++++++++++++++ frame/staking/src/voter_bags/mod.rs | 4 +- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 767192bd131ce..fe3d758d6b25a 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3860,6 +3860,87 @@ fn on_finalize_weight_is_nonzero() { }) } +#[test] +fn test_rebag() { + use crate::{ + testing_utils::create_stash_controller, + voter_bags::{Bag, Node}, + }; + use frame_benchmarking::{whitelisted_caller}; + use frame_system::RawOrigin; + + const USER_SEED: u32 = 999666; + + let whitelist_account = |account_id: &AccountIdOf| { + frame_benchmarking::benchmarking::add_to_whitelist( + frame_system::Account::::hashed_key_for(account_id).into() + ); + }; + + let make_validator = |n: u32, balance_factor: u32| -> Result<(AccountIdOf, AccountIdOf), &'static str> { + let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); + whitelist_account(&controller); + + let prefs = ValidatorPrefs::default(); + // bond the full value of the stash + Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); + Staking::validate(RawOrigin::Signed(controller.clone()).into(), prefs).unwrap(); + + Ok((stash, controller)) + }; + + ExtBuilder::default().build_and_execute(|| { + // stash controls the node account + let (stash, controller) = make_validator(USER_SEED, 100).unwrap(); + + // create another validator with 3x the stake + let (other_stash, _) = make_validator(USER_SEED + 1, 300).unwrap(); + + // update the stash account's value/weight + ::Currency::make_free_balance_be(&stash, ::Currency::free_balance(&other_stash)); + Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); + + // verify preconditions + let weight_of = Staking::weight_of_fn(); + let node = Node::::from_id(&stash).unwrap(); + assert!( + node.is_misplaced(&weight_of), + "rebagging only makes sense when a node is misplaced", + ); + assert_eq!( + { + let origin_bag = Bag::::get(node.bag_idx).unwrap(); + origin_bag.iter().count() + }, + 1, + "stash should be the only node in origin bag", + ); + let other_node = Node::::from_id(&other_stash).unwrap(); + assert!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); + assert_ne!( + { + let destination_bag = Bag::::get(other_node.bag_idx); + destination_bag.iter().count() + }, + 0, + "destination bag should not be empty", + ); + drop(node); + + // caller will call rebag + let caller = whitelisted_caller(); + // ensure it's distinct from the other accounts + assert_ne!(caller, stash); + assert_ne!(caller, controller); + + // call rebag + Pallet::::rebag(RawOrigin::Signed(caller).into(), stash.clone()).unwrap(); + + let node = Node::::from_id(&stash).unwrap(); + assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); + }); +} + mod election_data_provider { use super::*; use frame_election_provider_support::ElectionDataProvider; diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index ca65495faafb5..ee754dd795e97 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -390,7 +390,7 @@ impl Node { VoterType::Validator => Some(( self.voter.id.clone(), weight_of(&self.voter.id), - vec![self.voter.id.clone()], + sp_std::vec![self.voter.id.clone()], )), VoterType::Nominator => { let Nominations { submitted_in, mut targets, .. } = @@ -446,7 +446,7 @@ impl Node { /// Get the index of the bag that this node _should_ be in, given its vote weight. /// /// This is a helper intended only for benchmarking and should not be used in production. - #[cfg(feature = "runtime-benchmarks")] + #[cfg(any(test, feature = "runtime-benchmarks"))] pub fn proper_bag_for(&self) -> BagIdx { let weight_of = crate::Pallet::::weight_of_fn(); let current_weight = weight_of(&self.voter.id); From 1cb9e26b51be117d7339bed74cb4d2f59c776053 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 25 Jun 2021 16:27:04 +0200 Subject: [PATCH 28/82] WIP: rebag benchmark --- frame/staking/src/benchmarking.rs | 58 +++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index f9be1fb2f61bb..9abef89056ed8 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -647,45 +647,67 @@ benchmarks! { // - The destination bag is not empty, because then we need to update the `next` pointer // of the previous node in addition to the work we do otherwise. + use crate::voter_bags::{Bag, Node}; + + let dbg_weight = |human: &str, account_id: &T::AccountId| { + let weight_of = Staking::::weight_of_fn(); + sp_runtime::print(human); + sp_runtime::print(weight_of(account_id)); + }; + + let make_validator = |n: u32, balance_factor: u32| -> Result<(T::AccountId, T::AccountId), &'static str> { + let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default())?; + whitelist_account!(controller); + + let prefs = ValidatorPrefs::default(); + // bond the full value of the stash + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; + Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; + + Ok((stash, controller)) + }; + // Clean up any existing state. clear_validators_and_nominators::(); - use crate::voter_bags::{Bag, Node}; - // stash and controller control the node account, which is a validator for simplicity of setup - let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + // stash controls the node account + let (stash, controller) = make_validator(USER_SEED, 100)?; - let prefs = ValidatorPrefs::default(); - whitelist_account!(controller); - Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; + dbg_weight("stash", &stash); // create another validator with 3x the stake - create_validators::(1, 300)?; + let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; + + dbg_weight("other stash", &other_stash); // update the stash account's value/weight - T::Currency::deposit_into_existing(&stash, T::Currency::minimum_balance() * 200_u32.into())?; + T::Currency::make_free_balance_be(&stash, T::Currency::free_balance(&other_stash)); + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; + + dbg_weight("stash after deposit", &stash); // verify preconditions let weight_of = Staking::::weight_of_fn(); let node = Node::::from_id(&stash).ok_or("node not found for stash")?; - assert!( + ensure!( node.is_misplaced(&weight_of), "rebagging only makes sense when a node is misplaced", ); - assert_eq!( + ensure!( { let origin_bag = Bag::::get(node.bag_idx).ok_or("origin bag not found")?; - origin_bag.iter().count() + origin_bag.iter().count() == 1 }, - 1, "stash should be the only node in origin bag", ); - assert_ne!( + let other_node = Node::::from_id(&other_stash).ok_or("node not found for other_stash")?; + ensure!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); + ensure!( { let destination_bag = Bag::::get(node.proper_bag_for()).ok_or("destination bag not found")?; - destination_bag.iter().count() + destination_bag.iter().count() != 0 }, - 0, "destination bag should not be empty", ); drop(node); @@ -693,12 +715,12 @@ benchmarks! { // caller will call rebag let caller = whitelisted_caller(); // ensure it's distinct from the other accounts - assert_ne!(caller, stash); - assert_ne!(caller, controller); + ensure!(caller != stash, "caller must not be the same as the stash"); + ensure!(caller != controller, "caller must not be the same as the controller"); }: _(RawOrigin::Signed(caller), stash.clone()) verify { let node = Node::::from_id(&stash).ok_or("node not found for stash")?; - assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); + ensure!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); } } From 27cbb79e2c16999fea18863de0f0f47a23f6906f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 14:54:04 +0200 Subject: [PATCH 29/82] fix rebag benchmark After much trouble, it turns out that the reason this benchmark wasn't working properly was that it's not safe to assume that `!0_u32` is more than the actual free balance in an account. --- frame/staking/src/benchmarking.rs | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 4e82b40ab5c9a..98fd4bb50004e 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -649,19 +649,14 @@ benchmarks! { use crate::voter_bags::{Bag, Node}; - let dbg_weight = |human: &str, account_id: &T::AccountId| { - let weight_of = Staking::::weight_of_fn(); - sp_runtime::print(human); - sp_runtime::print(weight_of(account_id)); - }; - let make_validator = |n: u32, balance_factor: u32| -> Result<(T::AccountId, T::AccountId), &'static str> { let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default())?; whitelist_account!(controller); let prefs = ValidatorPrefs::default(); // bond the full value of the stash - Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; + let free_balance = T::Currency::free_balance(&stash); + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), free_balance)?; Staking::::validate(RawOrigin::Signed(controller.clone()).into(), prefs)?; Ok((stash, controller)) @@ -674,18 +669,13 @@ benchmarks! { // stash controls the node account let (stash, controller) = make_validator(USER_SEED, 100)?; - dbg_weight("stash", &stash); - // create another validator with 3x the stake let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; - dbg_weight("other stash", &other_stash); - // update the stash account's value/weight - T::Currency::make_free_balance_be(&stash, T::Currency::free_balance(&other_stash)); - Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into())?; - - dbg_weight("stash after deposit", &stash); + let other_free_balance = T::Currency::free_balance(&other_stash); + T::Currency::make_free_balance_be(&stash, other_free_balance); + Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), other_free_balance)?; // verify preconditions let weight_of = Staking::::weight_of_fn(); From b6508617781bd4efac9d767950dc8ba456bfcc7f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 28 Jun 2021 15:13:49 +0200 Subject: [PATCH 30/82] reduce diff --- frame/staking/src/lib.rs | 126 +++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index fd9ae7b101c8b..ba8e012c8643b 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -964,12 +964,12 @@ pub mod pallet { /// invulnerables) and restricted to testnets. #[pallet::storage] #[pallet::getter(fn invulnerables)] - pub type Invulnerables = StorageValue<_, Vec>, ValueQuery>; + pub type Invulnerables = StorageValue<_, Vec, ValueQuery>; /// Map from all locked "stash" accounts to the controller account. #[pallet::storage] #[pallet::getter(fn bonded)] - pub type Bonded = StorageMap<_, Twox64Concat, AccountIdOf, AccountIdOf>; + pub type Bonded = StorageMap<_, Twox64Concat, T::AccountId, T::AccountId>; /// The minimum active bond to become and maintain the role of a nominator. #[pallet::storage] @@ -984,8 +984,8 @@ pub mod pallet { #[pallet::getter(fn ledger)] pub type Ledger = StorageMap< _, - Blake2_128Concat, AccountIdOf, - StakingLedger, BalanceOf>, + Blake2_128Concat, T::AccountId, + StakingLedger>, >; /// Where the reward payment should be made. Keyed by stash. @@ -993,8 +993,8 @@ pub mod pallet { #[pallet::getter(fn payee)] pub type Payee = StorageMap< _, - Twox64Concat, AccountIdOf, - RewardDestination>, + Twox64Concat, T::AccountId, + RewardDestination, ValueQuery, >; @@ -1003,7 +1003,7 @@ pub mod pallet { /// When updating this storage item, you must also update the `CounterForValidators`. #[pallet::storage] #[pallet::getter(fn validators)] - pub type Validators = StorageMap<_, Twox64Concat, AccountIdOf, ValidatorPrefs, ValueQuery>; + pub type Validators = StorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; /// A tracker to keep count of the number of items in the `Validators` map. #[pallet::storage] @@ -1020,7 +1020,7 @@ pub mod pallet { /// When updating this storage item, you must also update the `CounterForNominators`. #[pallet::storage] #[pallet::getter(fn nominators)] - pub type Nominators = StorageMap<_, Twox64Concat, AccountIdOf, Nominations>>; + pub type Nominators = StorageMap<_, Twox64Concat, T::AccountId, Nominations>; /// A tracker to keep count of the number of items in the `Nominators` map. #[pallet::storage] @@ -1067,8 +1067,8 @@ pub mod pallet { pub type ErasStakers = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, - Exposure, BalanceOf>, + Twox64Concat, T::AccountId, + Exposure>, ValueQuery, >; @@ -1088,8 +1088,8 @@ pub mod pallet { pub type ErasStakersClipped = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, - Exposure, BalanceOf>, + Twox64Concat, T::AccountId, + Exposure>, ValueQuery, >; @@ -1104,7 +1104,7 @@ pub mod pallet { pub type ErasValidatorPrefs = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, + Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery, >; @@ -1123,7 +1123,7 @@ pub mod pallet { pub type ErasRewardPoints = StorageMap< _, Twox64Concat, EraIndex, - EraRewardPoints>, + EraRewardPoints, ValueQuery, >; @@ -1156,7 +1156,7 @@ pub mod pallet { pub type UnappliedSlashes = StorageMap< _, Twox64Concat, EraIndex, - Vec, BalanceOf>>, + Vec>>, ValueQuery, >; @@ -1173,7 +1173,7 @@ pub mod pallet { pub(crate) type ValidatorSlashInEra = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, + Twox64Concat, T::AccountId, (Perbill, BalanceOf), >; @@ -1182,20 +1182,20 @@ pub mod pallet { pub(crate) type NominatorSlashInEra = StorageDoubleMap< _, Twox64Concat, EraIndex, - Twox64Concat, AccountIdOf, + Twox64Concat, T::AccountId, BalanceOf, >; /// Slashing spans for stash accounts. #[pallet::storage] - pub(crate) type SlashingSpans = StorageMap<_, Twox64Concat, AccountIdOf, slashing::SlashingSpans>; + pub(crate) type SlashingSpans = StorageMap<_, Twox64Concat, T::AccountId, slashing::SlashingSpans>; /// Records information about the maximum slash of a stash within a slashing span, /// as well as how much reward has been paid out. #[pallet::storage] pub(crate) type SpanSlash = StorageMap< _, - Twox64Concat, (AccountIdOf, slashing::SpanIndex), + Twox64Concat, (T::AccountId, slashing::SpanIndex), slashing::SpanRecord>, ValueQuery, >; @@ -1264,7 +1264,7 @@ pub mod pallet { pub history_depth: u32, pub validator_count: u32, pub minimum_validator_count: u32, - pub invulnerables: Vec>, + pub invulnerables: Vec, pub force_era: Forcing, pub slash_reward_fraction: Perbill, pub canceled_payout: BalanceOf, @@ -1336,17 +1336,17 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] - #[pallet::metadata(AccountIdOf = "AccountId", BalanceOf = "Balance")] + #[pallet::metadata(T::AccountId = "AccountId", BalanceOf = "Balance")] pub enum Event { /// The era payout has been set; the first balance is the validator-payout; the second is /// the remainder from the maximum amount of reward. /// \[era_index, validator_payout, remainder\] EraPayout(EraIndex, BalanceOf, BalanceOf), /// The staker has been rewarded by this amount. \[stash, amount\] - Reward(AccountIdOf, BalanceOf), + Reward(T::AccountId, BalanceOf), /// One validator (and its nominators) has been slashed by the given amount. /// \[validator, amount\] - Slash(AccountIdOf, BalanceOf), + Slash(T::AccountId, BalanceOf), /// An old slashing report from a prior era was discarded because it could /// not be processed. \[session_index\] OldSlashingReportDiscarded(SessionIndex), @@ -1356,12 +1356,12 @@ pub mod pallet { /// /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, /// it will not be emitted for staking rewards when they are added to stake. - Bonded(AccountIdOf, BalanceOf), + Bonded(T::AccountId, BalanceOf), /// An account has unbonded this amount. \[stash, amount\] - Unbonded(AccountIdOf, BalanceOf), + Unbonded(T::AccountId, BalanceOf), /// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance` /// from the unlocking queue. \[stash, amount\] - Withdrawn(AccountIdOf, BalanceOf), + Withdrawn(T::AccountId, BalanceOf), /// A nominator has been kicked from a validator. \[nominator, stash\] Kicked(T::AccountId, T::AccountId), /// The election failed. No new era is planned. @@ -1504,7 +1504,7 @@ pub mod pallet { origin: OriginFor, controller: ::Source, #[pallet::compact] value: BalanceOf, - payee: RewardDestination>, + payee: RewardDestination, ) -> DispatchResult { let stash = ensure_signed(origin)?; @@ -1830,7 +1830,7 @@ pub mod pallet { } else { Err(Error::::BadTarget.into()) })) - .collect::>, _>>()?; + .collect::, _>>()?; let nominations = Nominations { targets, @@ -1889,7 +1889,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_payee())] pub fn set_payee( origin: OriginFor, - payee: RewardDestination>, + payee: RewardDestination, ) -> DispatchResult { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; @@ -2039,7 +2039,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::set_invulnerables(invulnerables.len() as u32))] pub fn set_invulnerables( origin: OriginFor, - invulnerables: Vec>, + invulnerables: Vec, ) -> DispatchResult { ensure_root(origin)?; >::put(invulnerables); @@ -2059,7 +2059,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::force_unstake(*num_slashing_spans))] pub fn force_unstake( origin: OriginFor, - stash: AccountIdOf, + stash: T::AccountId, num_slashing_spans: u32, ) -> DispatchResult { ensure_root(origin)?; @@ -2161,7 +2161,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::payout_stakers_alive_staked(T::MaxNominatorRewardedPerValidator::get()))] pub fn payout_stakers( origin: OriginFor, - validator_stash: AccountIdOf, + validator_stash: T::AccountId, era: EraIndex, ) -> DispatchResultWithPostInfo { ensure_signed(origin)?; @@ -2262,7 +2262,7 @@ pub mod pallet { #[pallet::weight(T::WeightInfo::reap_stash(*num_slashing_spans))] pub fn reap_stash( _origin: OriginFor, - stash: AccountIdOf, + stash: T::AccountId, num_slashing_spans: u32, ) -> DispatchResult { let at_minimum = T::Currency::total_balance(&stash) == T::Currency::minimum_balance(); @@ -2293,7 +2293,7 @@ pub mod pallet { for nom_stash in who.into_iter() .map(T::Lookup::lookup) - .collect::>, _>>()? + .collect::, _>>()? .into_iter() { Nominators::::mutate(&nom_stash, |maybe_nom| if let Some(ref mut nom) = maybe_nom { @@ -2437,14 +2437,14 @@ pub mod pallet { impl Pallet { /// The total balance that can be slashed from a stash account as of right now. - pub fn slashable_balance_of(stash: &AccountIdOf) -> BalanceOf { + pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { // Weight note: consider making the stake accessible through stash. Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() } /// Internal impl of [`Self::slashable_balance_of`] that returns [`VoteWeight`]. pub fn slashable_balance_of_vote_weight( - stash: &AccountIdOf, + stash: &T::AccountId, issuance: BalanceOf, ) -> VoteWeight { T::CurrencyToVote::to_vote(Self::slashable_balance_of(stash), issuance) @@ -2454,16 +2454,16 @@ impl Pallet { /// /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is /// important to be only used while the total issuance is not changing. - pub fn weight_of_fn() -> Box) -> VoteWeight> { + pub fn weight_of_fn() -> Box VoteWeight> { // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still // compile, while some types in mock fail to resolve. let issuance = T::Currency::total_issuance(); - Box::new(move |who: &AccountIdOf| -> VoteWeight { + Box::new(move |who: &T::AccountId| -> VoteWeight { Self::slashable_balance_of_vote_weight(who, issuance) }) } - fn do_payout_stakers(validator_stash: AccountIdOf, era: EraIndex) -> DispatchResultWithPostInfo { + fn do_payout_stakers(validator_stash: T::AccountId, era: EraIndex) -> DispatchResultWithPostInfo { // Validate input data let current_era = CurrentEra::::get().ok_or( Error::::InvalidEraToReward.with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) @@ -2579,8 +2579,8 @@ impl Pallet { /// /// This will also update the stash lock. fn update_ledger( - controller: &AccountIdOf, - ledger: &StakingLedger, BalanceOf> + controller: &T::AccountId, + ledger: &StakingLedger> ) { T::Currency::set_lock( STAKING_ID, @@ -2599,7 +2599,7 @@ impl Pallet { /// Actually make a payment to a staker. This uses the currency's reward function /// to pay the right payee for the given staker account. - fn make_payout(stash: &AccountIdOf, amount: BalanceOf) -> Option> { + fn make_payout(stash: &T::AccountId, amount: BalanceOf) -> Option> { let dest = Self::payee(stash); match dest { RewardDestination::Controller => Self::bonded(stash) @@ -2893,8 +2893,8 @@ impl Pallet { /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a /// [`Exposure`]. fn collect_exposures( - supports: Supports>, - ) -> Vec<(AccountIdOf, Exposure, BalanceOf>)> { + supports: Supports, + ) -> Vec<(T::AccountId, Exposure>)> { let total_issuance = T::Currency::total_issuance(); let to_currency = |e: frame_election_provider_support::ExtendedBalance| { T::CurrencyToVote::to_currency(e, total_issuance) @@ -2923,7 +2923,7 @@ impl Pallet { let exposure = Exposure { own, others, total }; (validator, exposure) }) - .collect::, Exposure<_, _>)>>() + .collect::)>>() } /// Remove all associated data of a stash account from the staking system. @@ -2933,7 +2933,7 @@ impl Pallet { /// This is called: /// - after a `withdraw_unbonded()` call that frees all of a stash's bonded balance. /// - through `reap_stash()` if the balance has fallen to zero (through slashing). - fn kill_stash(stash: &AccountIdOf, num_slashing_spans: u32) -> DispatchResult { + fn kill_stash(stash: &T::AccountId, num_slashing_spans: u32) -> DispatchResult { let controller = >::get(stash).ok_or(Error::::NotStash)?; slashing::clear_stash_metadata::(stash, num_slashing_spans)?; @@ -2991,7 +2991,7 @@ impl Pallet { /// /// COMPLEXITY: Complexity is `number_of_validator_to_reward x current_elected_len`. pub fn reward_by_ids( - validators_points: impl IntoIterator, u32)> + validators_points: impl IntoIterator ) { if let Some(active_era) = Self::active_era() { >::mutate(active_era.index, |era_rewards| { @@ -3014,8 +3014,8 @@ impl Pallet { #[cfg(feature = "runtime-benchmarks")] pub fn add_era_stakers( current_era: EraIndex, - controller: AccountIdOf, - exposure: Exposure, BalanceOf>, + controller: T::AccountId, + exposure: Exposure>, ) { >::insert(¤t_era, &controller, &exposure); } @@ -3112,7 +3112,7 @@ impl Pallet { } impl - frame_election_provider_support::ElectionDataProvider, BlockNumberFor> + frame_election_provider_support::ElectionDataProvider> for Pallet { const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; @@ -3184,8 +3184,8 @@ impl #[cfg(any(feature = "runtime-benchmarks", test))] fn put_snapshot( - voters: Vec<(AccountIdOf, VoteWeight, Vec>)>, - targets: Vec>, + voters: Vec<(T::AccountId, VoteWeight, Vec)>, + targets: Vec, target_stake: Option, ) { use sp_std::convert::TryFrom; @@ -3259,12 +3259,12 @@ impl pallet_session::SessionManager for Pallet { } } -impl historical::SessionManager, Exposure, BalanceOf>> +impl historical::SessionManager>> for Pallet { fn new_session( new_index: SessionIndex, - ) -> Option, Exposure, BalanceOf>)>> { + ) -> Option>)>> { >::new_session(new_index).map(|validators| { let current_era = Self::current_era() // Must be some as a new era has been created. @@ -3302,14 +3302,14 @@ impl historical::SessionManager, Exposure pallet_authorship::EventHandler, T::BlockNumber> for Pallet +impl pallet_authorship::EventHandler for Pallet where T: Config + pallet_authorship::Config + pallet_session::Config, { - fn note_author(author: AccountIdOf) { + fn note_author(author: T::AccountId) { Self::reward_by_ids(vec![(author, 20)]) } - fn note_uncle(author: AccountIdOf, _age: T::BlockNumber) { + fn note_uncle(author: T::AccountId, _age: T::BlockNumber) { Self::reward_by_ids(vec![ (>::author(), 2), (author, 1) @@ -3321,8 +3321,8 @@ where /// if any. pub struct StashOf(sp_std::marker::PhantomData); -impl Convert, Option>> for StashOf { - fn convert(controller: AccountIdOf) -> Option> { +impl Convert> for StashOf { + fn convert(controller: T::AccountId) -> Option { >::ledger(&controller).map(|l| l.stash) } } @@ -3334,10 +3334,10 @@ impl Convert, Option>> for StashOf { /// `active_era`. It can differ from the latest planned exposure in `current_era`. pub struct ExposureOf(sp_std::marker::PhantomData); -impl Convert, Option, BalanceOf>>> +impl Convert>>> for ExposureOf { - fn convert(validator: AccountIdOf) -> Option, BalanceOf>> { + fn convert(validator: T::AccountId) -> Option>> { >::active_era() .map(|active_era| >::eras_stakers(active_era.index, &validator)) } @@ -3345,7 +3345,7 @@ impl Convert, Option, BalanceO /// This is intended to be used with `FilterHistoricalOffences`. impl - OnOffenceHandler, pallet_session::historical::IdentificationTuple, Weight> + OnOffenceHandler, Weight> for Pallet where T: pallet_session::Config::AccountId>, @@ -3362,7 +3362,7 @@ where { fn on_offence( offenders: &[OffenceDetails< - AccountIdOf, + T::AccountId, pallet_session::historical::IdentificationTuple, >], slash_fraction: &[Perbill], From 9b53d4c5247327c942a00becc4bbf636828ed2d4 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 09:44:45 +0200 Subject: [PATCH 31/82] re-benchmark to add rebag --- frame/staking/src/lib.rs | 4 +- frame/staking/src/weights.rs | 307 ++++++++++++++++++----------------- 2 files changed, 161 insertions(+), 150 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index ba8e012c8643b..1788408030748 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -2412,9 +2412,7 @@ pub mod pallet { /// among the nominator/validator set once the snapshot is prepared for the election. /// /// Anyone can call this function about any stash. - // - // TODO: benchmark - #[pallet::weight(0)] + #[pallet::weight(T::WeightInfo::rebag())] pub fn rebag( origin: OriginFor, stash: AccountIdOf, diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index cf14e8b22362f..8f918a731fcd8 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-06-19, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-06-29, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -72,381 +72,394 @@ pub trait WeightInfo { fn get_npos_targets(v: u32, ) -> Weight; fn set_staking_limits() -> Weight; fn chill_other() -> Weight; + fn rebag() -> Weight; } /// Weights for pallet_staking using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn bond() -> Weight { - (72_617_000 as Weight) + (68_190_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (55_590_000 as Weight) + (52_644_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (59_730_000 as Weight) + (56_078_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (52_279_000 as Weight) + (49_356_000 as Weight) // Standard Error: 0 - .saturating_add((68_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (86_629_000 as Weight) + (91_998_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_379_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(8 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) + .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (32_393_000 as Weight) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (66_528_000 as Weight) + .saturating_add(T::DbWeight::get().reads(10 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (36_986_000 as Weight) - // Standard Error: 13_000 - .saturating_add((16_574_000 as Weight).saturating_mul(k as Weight)) + (39_441_000 as Weight) + // Standard Error: 18_000 + .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (43_228_000 as Weight) - // Standard Error: 21_000 - .saturating_add((5_119_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) + (87_415_000 as Weight) + // Standard Error: 42_000 + .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (17_800_000 as Weight) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + (27_259_000 as Weight) + .saturating_add(T::DbWeight::get().reads(5 as Weight)) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_612_000 as Weight) + (12_074_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_503_000 as Weight) + (27_101_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_119_000 as Weight) + (2_079_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_320_000 as Weight) + (2_249_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_269_000 as Weight) + (2_232_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_334_000 as Weight) + (2_240_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_354_000 as Weight) + (2_239_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (61_556_000 as Weight) + (72_352_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_377_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(6 as Weight)) - .saturating_add(T::DbWeight::get().writes(6 as Weight)) + .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(8 as Weight)) + .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_367_105_000 as Weight) - // Standard Error: 222_000 - .saturating_add((19_817_000 as Weight).saturating_mul(s as Weight)) + (4_702_953_000 as Weight) + // Standard Error: 309_000 + .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (47_229_000 as Weight) - // Standard Error: 53_000 - .saturating_add((48_365_000 as Weight).saturating_mul(n as Weight)) + (93_746_000 as Weight) + // Standard Error: 36_000 + .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (156_788_000 as Weight) - // Standard Error: 20_000 - .saturating_add((61_280_000 as Weight).saturating_mul(n as Weight)) + (109_819_000 as Weight) + // Standard Error: 58_000 + .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (47_815_000 as Weight) - // Standard Error: 1_000 - .saturating_add((65_000 as Weight).saturating_mul(l as Weight)) + (48_053_000 as Weight) + // Standard Error: 2_000 + .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) // Standard Error: 74_000 - .saturating_add((34_945_000 as Weight).saturating_mul(e as Weight)) + .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (73_483_000 as Weight) - // Standard Error: 0 - .saturating_add((2_384_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(7 as Weight)) - .saturating_add(T::DbWeight::get().writes(8 as Weight)) + (104_449_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 846_000 - .saturating_add((305_234_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((48_280_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(T::DbWeight::get().reads(10 as Weight)) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + // Standard Error: 1_056_000 + .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 53_000 + .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(83 as Weight)) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 99_000 - .saturating_add((25_735_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 99_000 - .saturating_add((28_122_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_388_000 - .saturating_add((21_500_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(T::DbWeight::get().reads(3 as Weight)) + // Standard Error: 100_000 + .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 100_000 + .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_411_000 + .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(T::DbWeight::get().reads(75 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 30_000 - .saturating_add((11_065_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 32_000 + .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_028_000 as Weight) - .saturating_add(T::DbWeight::get().writes(4 as Weight)) + (5_669_000 as Weight) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (35_758_000 as Weight) - .saturating_add(T::DbWeight::get().reads(5 as Weight)) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) + (72_656_000 as Weight) + .saturating_add(T::DbWeight::get().reads(11 as Weight)) + .saturating_add(T::DbWeight::get().writes(6 as Weight)) + } + fn rebag() -> Weight { + (83_208_000 as Weight) + .saturating_add(T::DbWeight::get().reads(7 as Weight)) + .saturating_add(T::DbWeight::get().writes(5 as Weight)) } } // For backwards compatibility and tests impl WeightInfo for () { fn bond() -> Weight { - (72_617_000 as Weight) + (68_190_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (55_590_000 as Weight) + (52_644_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (59_730_000 as Weight) + (56_078_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (52_279_000 as Weight) + (49_356_000 as Weight) // Standard Error: 0 - .saturating_add((68_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (86_629_000 as Weight) + (91_998_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_379_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(8 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (32_393_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (66_528_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(10 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (36_986_000 as Weight) - // Standard Error: 13_000 - .saturating_add((16_574_000 as Weight).saturating_mul(k as Weight)) + (39_441_000 as Weight) + // Standard Error: 18_000 + .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (43_228_000 as Weight) - // Standard Error: 21_000 - .saturating_add((5_119_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + (87_415_000 as Weight) + // Standard Error: 42_000 + .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (17_800_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + (27_259_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(5 as Weight)) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_612_000 as Weight) + (12_074_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_503_000 as Weight) + (27_101_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_119_000 as Weight) + (2_079_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_320_000 as Weight) + (2_249_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_269_000 as Weight) + (2_232_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_334_000 as Weight) + (2_240_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_354_000 as Weight) + (2_239_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (61_556_000 as Weight) + (72_352_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_377_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(6 as Weight)) - .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(8 as Weight)) + .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (3_367_105_000 as Weight) - // Standard Error: 222_000 - .saturating_add((19_817_000 as Weight).saturating_mul(s as Weight)) + (4_702_953_000 as Weight) + // Standard Error: 309_000 + .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (47_229_000 as Weight) - // Standard Error: 53_000 - .saturating_add((48_365_000 as Weight).saturating_mul(n as Weight)) + (93_746_000 as Weight) + // Standard Error: 36_000 + .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (156_788_000 as Weight) - // Standard Error: 20_000 - .saturating_add((61_280_000 as Weight).saturating_mul(n as Weight)) + (109_819_000 as Weight) + // Standard Error: 58_000 + .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (47_815_000 as Weight) - // Standard Error: 1_000 - .saturating_add((65_000 as Weight).saturating_mul(l as Weight)) + (48_053_000 as Weight) + // Standard Error: 2_000 + .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) // Standard Error: 74_000 - .saturating_add((34_945_000 as Weight).saturating_mul(e as Weight)) + .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (73_483_000 as Weight) - // Standard Error: 0 - .saturating_add((2_384_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(7 as Weight)) - .saturating_add(RocksDbWeight::get().writes(8 as Weight)) + (104_449_000 as Weight) + // Standard Error: 1_000 + .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 846_000 - .saturating_add((305_234_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 42_000 - .saturating_add((48_280_000 as Weight).saturating_mul(n as Weight)) - .saturating_add(RocksDbWeight::get().reads(10 as Weight)) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + // Standard Error: 1_056_000 + .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 53_000 + .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(83 as Weight)) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(v as Weight))) } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 99_000 - .saturating_add((25_735_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 99_000 - .saturating_add((28_122_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_388_000 - .saturating_add((21_500_000 as Weight).saturating_mul(s as Weight)) - .saturating_add(RocksDbWeight::get().reads(3 as Weight)) + // Standard Error: 100_000 + .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 100_000 + .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 3_411_000 + .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + .saturating_add(RocksDbWeight::get().reads(75 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) - .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(s as Weight))) } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 30_000 - .saturating_add((11_065_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 32_000 + .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_028_000 as Weight) - .saturating_add(RocksDbWeight::get().writes(4 as Weight)) + (5_669_000 as Weight) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (35_758_000 as Weight) - .saturating_add(RocksDbWeight::get().reads(5 as Weight)) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + (72_656_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(11 as Weight)) + .saturating_add(RocksDbWeight::get().writes(6 as Weight)) + } + fn rebag() -> Weight { + (83_208_000 as Weight) + .saturating_add(RocksDbWeight::get().reads(7 as Weight)) + .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } } From ab048c6ca0b6bc0673f3f11a7d4205c34ba0457a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 15:31:19 +0200 Subject: [PATCH 32/82] add VoterList::regenerate() --- frame/staking/src/voter_bags/mod.rs | 37 +++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index ee754dd795e97..ecacd1ebe85ca 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -25,8 +25,8 @@ mod thresholds; use thresholds::THRESHOLDS; use crate::{ - AccountIdOf, Config, Nominations, Nominators, Pallet, VoteWeight, VoterBagFor, VotingDataOf, - slashing::SlashingSpans, + AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, + VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; use frame_support::DefaultNoBound; @@ -81,6 +81,23 @@ impl VoterList { crate::VoterNodes::::remove_all(None); } + /// Regenerate voter data from the `Nominators` and `Validators` storage items. + /// + /// This is expensive and should only ever be performed during a migration, never during + /// consensus. + pub fn regenerate() { + Self::clear(); + + let nominators_iter = Nominators::::iter().map(|(id, _)| Voter::nominator(id)); + let validators_iter = Validators::::iter().map(|(id, _)| Voter::validator(id)); + let weight_of = Pallet::::weight_of_fn(); + + Self::insert_many( + nominators_iter.chain(validators_iter), + weight_of, + ); + } + /// Decode the length of the voter list. pub fn decode_len() -> Option { crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) @@ -464,6 +481,22 @@ pub struct Voter { pub voter_type: VoterType, } +impl Voter { + pub fn nominator(id: AccountId) -> Self { + Self { + id, + voter_type: VoterType::Nominator, + } + } + + pub fn validator(id: AccountId) -> Self { + Self { + id, + voter_type: VoterType::Validator, + } + } +} + pub type VoterOf = Voter>; /// Type of voter. From 5583ae4fd712a1ddd37a3cb173e11459e92167aa Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 16:23:24 +0200 Subject: [PATCH 33/82] add VoterList migration --- frame/staking/src/lib.rs | 32 ++++++++++++++++++++++++++++- frame/staking/src/voter_bags/mod.rs | 11 ++++++---- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 1788408030748..ba35473385540 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -748,17 +748,47 @@ enum Releases { V5_0_0, // blockable validators. V6_0_0, // removal of all storage associated with offchain phragmen. V7_0_0, // keep track of number of nominators / validators in map + V8_0_0, // VoterList and efficient semi-sorted iteration } impl Default for Releases { fn default() -> Self { - Releases::V7_0_0 + Releases::V8_0_0 } } pub mod migrations { use super::*; + pub mod v8 { + use super::*; + + pub fn pre_migrate() -> Result<(), &'static str> { + ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); + ensure!(VoterList::::decode_len().unwrap_or_default() == 0, "voter list already exists"); + Ok(()) + } + + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V8_0_0"); + + let migrated = VoterList::::regenerate(); + + StorageVersion::::put(Releases::V8_0_0); + log!( + info, + "Completed staking migration to Releases::V8_0_0 with {} voters migrated", + migrated, + ); + + // TODO: this is a pretty rough estimate, improve it + T::DbWeight::get().reads_writes( + migrated.into(), + (migrated * 3).into(), + ) + } + } + pub mod v7 { use super::*; diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index ecacd1ebe85ca..626ff010573f3 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -85,7 +85,9 @@ impl VoterList { /// /// This is expensive and should only ever be performed during a migration, never during /// consensus. - pub fn regenerate() { + /// + /// Returns the number of voters migrated. + pub fn regenerate() -> u32 { Self::clear(); let nominators_iter = Nominators::::iter().map(|(id, _)| Voter::nominator(id)); @@ -95,7 +97,7 @@ impl VoterList { Self::insert_many( nominators_iter.chain(validators_iter), weight_of, - ); + ) } /// Decode the length of the voter list. @@ -127,7 +129,7 @@ impl VoterList { /// Insert a new voter into the appropriate bag in the voter list. pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { - Self::insert_many(sp_std::iter::once(voter), weight_of) + Self::insert_many(sp_std::iter::once(voter), weight_of); } /// Insert several voters into the appropriate bags in the voter list. @@ -136,7 +138,7 @@ impl VoterList { pub fn insert_many( voters: impl IntoIterator>, weight_of: impl Fn(&T::AccountId) -> VoteWeight, - ) { + ) -> u32 { let mut bags = BTreeMap::new(); let mut count = 0; @@ -152,6 +154,7 @@ impl VoterList { } crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); + count } /// Remove a voter (by id) from the voter list. From ae67ae79eb9d494b33c04d95d74fa6ac91f0ad6a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 10:42:38 +0200 Subject: [PATCH 34/82] add benchmark for regenerate --- frame/staking/src/benchmarking.rs | 22 +++ frame/staking/src/lib.rs | 9 +- frame/staking/src/weights.rs | 249 +++++++++++++++++------------- 3 files changed, 164 insertions(+), 116 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 2d23a6c983f0d..6f2cd2e7d07a1 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -719,6 +719,28 @@ benchmarks! { let node = Node::::from_id(&stash).ok_or("node not found for stash")?; ensure!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); } + + regenerate { + // number of validator intention. + let v in (MAX_VALIDATORS / 2) .. MAX_VALIDATORS; + // number of nominator intention. + let n in (MAX_NOMINATORS / 2) .. MAX_NOMINATORS; + + clear_validators_and_nominators::(); + ensure!( + create_validators_with_nominators_for_era::( + v, + n, + T::MAX_NOMINATIONS as usize, + true, + None, + ).is_ok(), + "creating validators and nominators failed", + ); + }: { + let migrated = VoterList::::regenerate(); + ensure!(v + n == migrated, "didn't migrate right amount of voters"); + } } #[cfg(test)] diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index ba35473385540..77b59e977385a 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -781,11 +781,10 @@ pub mod migrations { migrated, ); - // TODO: this is a pretty rough estimate, improve it - T::DbWeight::get().reads_writes( - migrated.into(), - (migrated * 3).into(), - ) + T::WeightInfo::regenerate( + CounterForValidators::::get(), + CounterForNominators::::get(), + ).saturating_add(T::DbWeight::get().reads(2)) } } diff --git a/frame/staking/src/weights.rs b/frame/staking/src/weights.rs index 8f918a731fcd8..1a259e4966082 100644 --- a/frame/staking/src/weights.rs +++ b/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for pallet_staking //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 3.0.0 -//! DATE: 2021-06-29, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2021-06-30, STEPS: `[50, ]`, REPEAT: 20, LOW RANGE: `[]`, HIGH RANGE: `[]` //! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 128 // Executed Command: @@ -73,161 +73,162 @@ pub trait WeightInfo { fn set_staking_limits() -> Weight; fn chill_other() -> Weight; fn rebag() -> Weight; + fn regenerate(v: u32, n: u32, ) -> Weight; } /// Weights for pallet_staking using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { fn bond() -> Weight { - (68_190_000 as Weight) + (70_861_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (52_644_000 as Weight) + (54_634_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (56_078_000 as Weight) + (58_152_000 as Weight) .saturating_add(T::DbWeight::get().reads(6 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (49_356_000 as Weight) + (53_561_000 as Weight) // Standard Error: 0 - .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((38_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(4 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (91_998_000 as Weight) + (93_297_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_349_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (66_528_000 as Weight) + (67_912_000 as Weight) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (39_441_000 as Weight) - // Standard Error: 18_000 - .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) + (44_238_000 as Weight) + // Standard Error: 20_000 + .saturating_add((15_852_000 as Weight).saturating_mul(k as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (87_415_000 as Weight) - // Standard Error: 42_000 - .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + (91_375_000 as Weight) + // Standard Error: 48_000 + .saturating_add((5_002_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(12 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (27_259_000 as Weight) + (27_150_000 as Weight) .saturating_add(T::DbWeight::get().reads(5 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_074_000 as Weight) + (11_982_000 as Weight) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_101_000 as Weight) + (26_858_000 as Weight) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_079_000 as Weight) + (2_000_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_249_000 as Weight) + (2_232_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_232_000 as Weight) + (2_256_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_240_000 as Weight) + (2_224_000 as Weight) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_239_000 as Weight) + (2_344_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (72_352_000 as Weight) + (70_433_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_352_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(8 as Weight)) .saturating_add(T::DbWeight::get().writes(7 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (4_702_953_000 as Weight) - // Standard Error: 309_000 - .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) + (3_208_783_000 as Weight) + // Standard Error: 220_000 + .saturating_add((18_751_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (93_746_000 as Weight) - // Standard Error: 36_000 - .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) + (136_667_000 as Weight) + // Standard Error: 31_000 + .saturating_add((47_790_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(10 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(2 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (109_819_000 as Weight) - // Standard Error: 58_000 - .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) + (157_202_000 as Weight) + // Standard Error: 36_000 + .saturating_add((61_104_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(T::DbWeight::get().writes(3 as Weight)) .saturating_add(T::DbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (48_053_000 as Weight) + (48_267_000 as Weight) // Standard Error: 2_000 - .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((28_000 as Weight).saturating_mul(l as Weight)) .saturating_add(T::DbWeight::get().reads(3 as Weight)) .saturating_add(T::DbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 74_000 - .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 71_000 + .saturating_add((34_769_000 as Weight).saturating_mul(e as Weight)) .saturating_add(T::DbWeight::get().reads(2 as Weight)) .saturating_add(T::DbWeight::get().writes(4 as Weight)) .saturating_add(T::DbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (104_449_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + (108_325_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_365_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(12 as Weight)) .saturating_add(T::DbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 1_056_000 - .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 53_000 - .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 1_084_000 + .saturating_add((307_737_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 54_000 + .saturating_add((51_644_000 as Weight).saturating_mul(n as Weight)) .saturating_add(T::DbWeight::get().reads(83 as Weight)) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -236,12 +237,12 @@ impl WeightInfo for SubstrateWeight { } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 100_000 - .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 100_000 - .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_411_000 - .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 118_000 + .saturating_add((21_454_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 118_000 + .saturating_add((31_494_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 4_041_000 + .saturating_add((52_930_000 as Weight).saturating_mul(s as Weight)) .saturating_add(T::DbWeight::get().reads(75 as Weight)) .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(T::DbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -249,179 +250,192 @@ impl WeightInfo for SubstrateWeight { } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 32_000 - .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 29_000 + .saturating_add((10_880_000 as Weight).saturating_mul(v as Weight)) .saturating_add(T::DbWeight::get().reads(1 as Weight)) .saturating_add(T::DbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_669_000 as Weight) + (5_902_000 as Weight) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (72_656_000 as Weight) + (73_685_000 as Weight) .saturating_add(T::DbWeight::get().reads(11 as Weight)) .saturating_add(T::DbWeight::get().writes(6 as Weight)) } fn rebag() -> Weight { - (83_208_000 as Weight) + (84_501_000 as Weight) .saturating_add(T::DbWeight::get().reads(7 as Weight)) .saturating_add(T::DbWeight::get().writes(5 as Weight)) } + fn regenerate(v: u32, n: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 128_000 + .saturating_add((40_328_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 128_000 + .saturating_add((42_763_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) + .saturating_add(T::DbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(T::DbWeight::get().writes(14 as Weight)) + .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(v as Weight))) + .saturating_add(T::DbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) + } } // For backwards compatibility and tests impl WeightInfo for () { fn bond() -> Weight { - (68_190_000 as Weight) + (70_861_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) } fn bond_extra() -> Weight { - (52_644_000 as Weight) + (54_634_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) } fn unbond() -> Weight { - (56_078_000 as Weight) + (58_152_000 as Weight) .saturating_add(RocksDbWeight::get().reads(6 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_update(s: u32, ) -> Weight { - (49_356_000 as Weight) + (53_561_000 as Weight) // Standard Error: 0 - .saturating_add((72_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((38_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(4 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn withdraw_unbonded_kill(s: u32, ) -> Weight { - (91_998_000 as Weight) + (93_297_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_272_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_349_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn validate() -> Weight { - (66_528_000 as Weight) + (67_912_000 as Weight) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn kick(k: u32, ) -> Weight { - (39_441_000 as Weight) - // Standard Error: 18_000 - .saturating_add((15_690_000 as Weight).saturating_mul(k as Weight)) + (44_238_000 as Weight) + // Standard Error: 20_000 + .saturating_add((15_852_000 as Weight).saturating_mul(k as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(k as Weight))) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(k as Weight))) } fn nominate(n: u32, ) -> Weight { - (87_415_000 as Weight) - // Standard Error: 42_000 - .saturating_add((5_409_000 as Weight).saturating_mul(n as Weight)) + (91_375_000 as Weight) + // Standard Error: 48_000 + .saturating_add((5_002_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(12 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) } fn chill() -> Weight { - (27_259_000 as Weight) + (27_150_000 as Weight) .saturating_add(RocksDbWeight::get().reads(5 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_payee() -> Weight { - (12_074_000 as Weight) + (11_982_000 as Weight) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_controller() -> Weight { - (27_101_000 as Weight) + (26_858_000 as Weight) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_validator_count() -> Weight { - (2_079_000 as Weight) + (2_000_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_no_eras() -> Weight { - (2_249_000 as Weight) + (2_232_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era() -> Weight { - (2_232_000 as Weight) + (2_256_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_new_era_always() -> Weight { - (2_240_000 as Weight) + (2_224_000 as Weight) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn set_invulnerables(v: u32, ) -> Weight { - (2_239_000 as Weight) + (2_344_000 as Weight) // Standard Error: 0 .saturating_add((5_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn force_unstake(s: u32, ) -> Weight { - (72_352_000 as Weight) + (70_433_000 as Weight) // Standard Error: 1_000 - .saturating_add((2_243_000 as Weight).saturating_mul(s as Weight)) + .saturating_add((2_352_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(8 as Weight)) .saturating_add(RocksDbWeight::get().writes(7 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn cancel_deferred_slash(s: u32, ) -> Weight { - (4_702_953_000 as Weight) - // Standard Error: 309_000 - .saturating_add((27_740_000 as Weight).saturating_mul(s as Weight)) + (3_208_783_000 as Weight) + // Standard Error: 220_000 + .saturating_add((18_751_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().writes(1 as Weight)) } fn payout_stakers_dead_controller(n: u32, ) -> Weight { - (93_746_000 as Weight) - // Standard Error: 36_000 - .saturating_add((46_832_000 as Weight).saturating_mul(n as Weight)) + (136_667_000 as Weight) + // Standard Error: 31_000 + .saturating_add((47_790_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(10 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(2 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(n as Weight))) } fn payout_stakers_alive_staked(n: u32, ) -> Weight { - (109_819_000 as Weight) - // Standard Error: 58_000 - .saturating_add((60_675_000 as Weight).saturating_mul(n as Weight)) + (157_202_000 as Weight) + // Standard Error: 36_000 + .saturating_add((61_104_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().reads((5 as Weight).saturating_mul(n as Weight))) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) .saturating_add(RocksDbWeight::get().writes((3 as Weight).saturating_mul(n as Weight))) } fn rebond(l: u32, ) -> Weight { - (48_053_000 as Weight) + (48_267_000 as Weight) // Standard Error: 2_000 - .saturating_add((33_000 as Weight).saturating_mul(l as Weight)) + .saturating_add((28_000 as Weight).saturating_mul(l as Weight)) .saturating_add(RocksDbWeight::get().reads(3 as Weight)) .saturating_add(RocksDbWeight::get().writes(3 as Weight)) } fn set_history_depth(e: u32, ) -> Weight { (0 as Weight) - // Standard Error: 74_000 - .saturating_add((34_228_000 as Weight).saturating_mul(e as Weight)) + // Standard Error: 71_000 + .saturating_add((34_769_000 as Weight).saturating_mul(e as Weight)) .saturating_add(RocksDbWeight::get().reads(2 as Weight)) .saturating_add(RocksDbWeight::get().writes(4 as Weight)) .saturating_add(RocksDbWeight::get().writes((7 as Weight).saturating_mul(e as Weight))) } fn reap_stash(s: u32, ) -> Weight { - (104_449_000 as Weight) - // Standard Error: 1_000 - .saturating_add((2_279_000 as Weight).saturating_mul(s as Weight)) + (108_325_000 as Weight) + // Standard Error: 2_000 + .saturating_add((2_365_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(12 as Weight)) .saturating_add(RocksDbWeight::get().writes((1 as Weight).saturating_mul(s as Weight))) } fn new_era(v: u32, n: u32, ) -> Weight { (0 as Weight) - // Standard Error: 1_056_000 - .saturating_add((304_507_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 53_000 - .saturating_add((51_152_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 1_084_000 + .saturating_add((307_737_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 54_000 + .saturating_add((51_644_000 as Weight).saturating_mul(n as Weight)) .saturating_add(RocksDbWeight::get().reads(83 as Weight)) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -430,12 +444,12 @@ impl WeightInfo for () { } fn get_npos_voters(v: u32, n: u32, s: u32, ) -> Weight { (0 as Weight) - // Standard Error: 100_000 - .saturating_add((22_603_000 as Weight).saturating_mul(v as Weight)) - // Standard Error: 100_000 - .saturating_add((32_482_000 as Weight).saturating_mul(n as Weight)) - // Standard Error: 3_411_000 - .saturating_add((45_781_000 as Weight).saturating_mul(s as Weight)) + // Standard Error: 118_000 + .saturating_add((21_454_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 118_000 + .saturating_add((31_494_000 as Weight).saturating_mul(n as Weight)) + // Standard Error: 4_041_000 + .saturating_add((52_930_000 as Weight).saturating_mul(s as Weight)) .saturating_add(RocksDbWeight::get().reads(75 as Weight)) .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) .saturating_add(RocksDbWeight::get().reads((4 as Weight).saturating_mul(n as Weight))) @@ -443,23 +457,36 @@ impl WeightInfo for () { } fn get_npos_targets(v: u32, ) -> Weight { (0 as Weight) - // Standard Error: 32_000 - .saturating_add((10_841_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 29_000 + .saturating_add((10_880_000 as Weight).saturating_mul(v as Weight)) .saturating_add(RocksDbWeight::get().reads(1 as Weight)) .saturating_add(RocksDbWeight::get().reads((1 as Weight).saturating_mul(v as Weight))) } fn set_staking_limits() -> Weight { - (5_669_000 as Weight) + (5_902_000 as Weight) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } fn chill_other() -> Weight { - (72_656_000 as Weight) + (73_685_000 as Weight) .saturating_add(RocksDbWeight::get().reads(11 as Weight)) .saturating_add(RocksDbWeight::get().writes(6 as Weight)) } fn rebag() -> Weight { - (83_208_000 as Weight) + (84_501_000 as Weight) .saturating_add(RocksDbWeight::get().reads(7 as Weight)) .saturating_add(RocksDbWeight::get().writes(5 as Weight)) } + fn regenerate(v: u32, n: u32, ) -> Weight { + (0 as Weight) + // Standard Error: 128_000 + .saturating_add((40_328_000 as Weight).saturating_mul(v as Weight)) + // Standard Error: 128_000 + .saturating_add((42_763_000 as Weight).saturating_mul(n as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(v as Weight))) + .saturating_add(RocksDbWeight::get().reads((3 as Weight).saturating_mul(n as Weight))) + .saturating_add(RocksDbWeight::get().writes(14 as Weight)) + .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(v as Weight))) + .saturating_add(RocksDbWeight::get().writes((2 as Weight).saturating_mul(n as Weight))) + } } From efe15e22261c440adc265a64b6c87d3677fdb038 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:42:13 +0200 Subject: [PATCH 35/82] remove redundant check --- frame/staking/src/benchmarking.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 6f2cd2e7d07a1..af03448204f1e 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -103,7 +103,6 @@ pub fn create_validator_with_nominators( ); assert_ne!(CounterForValidators::::get(), 0); assert_ne!(CounterForNominators::::get(), 0); - assert_ne!(VoterList::::decode_len().unwrap_or_default(), 0); // Give Era Points let reward = EraRewardPoints:: { From fb5431e1782c60a2c59d0f7fa3a5c752c73bed35 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:45:25 +0200 Subject: [PATCH 36/82] debug assertions that `VoterList` and `CounterFor{Nominators,Validators}` are in sync Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/staking/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 77b59e977385a..6d739411007c8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -3103,6 +3103,7 @@ impl Pallet { } Nominators::::insert(who, nominations); VoterList::::insert_as(who, VoterType::Nominator); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } /// This function will remove a nominator from the `Nominators` storage map, @@ -3112,6 +3113,7 @@ impl Pallet { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } } @@ -3125,6 +3127,7 @@ impl Pallet { } Validators::::insert(who, prefs); VoterList::::insert_as(who, VoterType::Validator); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } /// This function will remove a validator from the `Validators` storage map, @@ -3134,6 +3137,7 @@ impl Pallet { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); + debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } } } From ee7959db5ff6bd1c80a81078c99dcd53bd0dc223 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:47:24 +0200 Subject: [PATCH 37/82] remove extra whitespace Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> --- frame/staking/src/benchmarking.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index af03448204f1e..39338fb518e4d 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -671,7 +671,6 @@ benchmarks! { // Clean up any existing state. clear_validators_and_nominators::(); - // stash controls the node account let (stash, controller) = make_validator(USER_SEED, 100)?; From f58bb3aa006d75a49056219dc7f4123bce75cebc Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:50:53 +0200 Subject: [PATCH 38/82] only emit rebag event on success --- frame/staking/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 6d739411007c8..3b4cff10f8feb 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1397,8 +1397,8 @@ pub mod pallet { StakingElectionFailed, /// Attempted to rebag an account. /// - /// If the second parameter is not `None`, it is the `(from, to)` tuple of bag indices. - Rebag(T::AccountId, Option<(BagIdx, BagIdx)>), + /// Contents: `who, from, to`. + Rebagged(T::AccountId, BagIdx, BagIdx), } #[pallet::error] @@ -2450,12 +2450,12 @@ pub mod pallet { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. - let moved = voter_bags::Node::::from_id(&stash).and_then(|node| { + if let Some((from, to)) = voter_bags::Node::::from_id(&stash).and_then(|node| { let weight_of = Self::weight_of_fn(); VoterList::update_position_for(node, weight_of) - }); - - Self::deposit_event(Event::::Rebag(stash, moved)); + }) { + Self::deposit_event(Event::::Rebagged(stash, from, to)); + }; Ok(()) } From 3c92edac421433656543d04a7d60e815d41b1112 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 11:55:44 +0200 Subject: [PATCH 39/82] add doc explaining the term voter --- frame/staking/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 3b4cff10f8feb..c2c8dbbb36489 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -100,6 +100,13 @@ //! //! An account can become a nominator via the [`nominate`](Call::nominate) call. //! +//! #### Voting +//! +//! Staking is closely related to elections; actual validators are chosen from among all potential +//! validators by election by the potential validators and nominators. To reduce use of the phrase +//! "potential validators and nominators", we often use the term **voters**, who are simply +//! the union of potential validators and nominators. +//! //! #### Rewards and Slash //! //! The **reward and slashing** procedure is the core of the Staking pallet, attempting to _embrace From 22bc615a25c1f76ac42c1060061b74f9ab644fda Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 13:23:20 +0200 Subject: [PATCH 40/82] revamp/simplify rebag test --- frame/staking/src/tests.rs | 60 ++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index d76aed9ff04f4..a41886d8b37c8 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3863,42 +3863,42 @@ fn on_finalize_weight_is_nonzero() { #[test] fn test_rebag() { use crate::{ - testing_utils::create_stash_controller, + testing_utils::{create_funded_user, create_stash_controller}, voter_bags::{Bag, Node}, }; - use frame_benchmarking::{whitelisted_caller}; use frame_system::RawOrigin; - const USER_SEED: u32 = 999666; - - let whitelist_account = |account_id: &AccountIdOf| { - frame_benchmarking::benchmarking::add_to_whitelist( - frame_system::Account::::hashed_key_for(account_id).into() - ); - }; - - let make_validator = |n: u32, balance_factor: u32| -> Result<(AccountIdOf, AccountIdOf), &'static str> { + fn make_validator(n: u32, balance_factor: u32) -> Result<(AccountIdOf, AccountIdOf), &'static str> { let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); - whitelist_account(&controller); - let prefs = ValidatorPrefs::default(); - // bond the full value of the stash - Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); - Staking::validate(RawOrigin::Signed(controller.clone()).into(), prefs).unwrap(); + // Bond the full value of the stash + // + // By default, `create_stash_controller` only bonds 10% of the stash. However, we're going + // to want to edit one account's bonded value to match another's, so it's simpler if 100% of + // the balance is bonded. + let balance = ::Currency::free_balance(&stash); + Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), balance).unwrap(); + Staking::validate( + RawOrigin::Signed(controller.clone()).into(), + ValidatorPrefs::default(), + ).unwrap(); Ok((stash, controller)) - }; + } ExtBuilder::default().build_and_execute(|| { - // stash controls the node account - let (stash, controller) = make_validator(USER_SEED, 100).unwrap(); - - // create another validator with 3x the stake - let (other_stash, _) = make_validator(USER_SEED + 1, 300).unwrap(); - - // update the stash account's value/weight - ::Currency::make_free_balance_be(&stash, ::Currency::free_balance(&other_stash)); - Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), (!0_u32).into()).unwrap(); + // We want to have two validators: one, `stash`, is the one we will rebag. + // The other, `other_stash`, exists only so that the destination bag is not empty. + let (stash, controller) = make_validator(0, 100).unwrap(); + let (other_stash, _) = make_validator(1, 300).unwrap(); + + // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. + let new_balance = ::Currency::free_balance(&other_stash); + ::Currency::make_free_balance_be(&stash, new_balance); + Staking::bond_extra( + RawOrigin::Signed(stash.clone()).into(), + new_balance, + ).unwrap(); // verify preconditions let weight_of = Staking::weight_of_fn(); @@ -3925,17 +3925,19 @@ fn test_rebag() { 0, "destination bag should not be empty", ); - drop(node); - // caller will call rebag - let caller = whitelisted_caller(); + // Any unrelated person can call `rebag`, as long as they sign and pay the tx fee. + let caller = create_funded_user::("caller", 3, 100); // ensure it's distinct from the other accounts assert_ne!(caller, stash); + assert_ne!(caller, other_stash); assert_ne!(caller, controller); // call rebag Pallet::::rebag(RawOrigin::Signed(caller).into(), stash.clone()).unwrap(); + // node should no longer be misplaced + // note that we have to refresh the node let node = Node::::from_id(&stash).unwrap(); assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); }); From 4092fa8d50741c4195c0039af30b5bd302e9d5b3 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 14:16:58 +0200 Subject: [PATCH 41/82] ensure genesis accounts are placed into the correct nodes/bags --- frame/staking/src/tests.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index a41886d8b37c8..fa31e84ae47d3 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -147,6 +147,17 @@ fn basic_setup_works() { // New era is not being forced assert_eq!(Staking::force_era(), Forcing::NotForcing); + + // genesis accounts must exist in the proper bags + let weight_of = Staking::weight_of_fn(); + // for these stash ids, see + // https://github.com/paritytech/substrate/ + // blob/631d4cdbcad438248c2597213918d8207d85bf6e/frame/staking/src/mock.rs#L435-L441 + for genesis_stash_account_id in [11, 21, 31, 101] { + let node = crate::voter_bags::Node::::from_id(&genesis_stash_account_id) + .expect(&format!("node was created for account {}", genesis_stash_account_id)); + assert!(!node.is_misplaced(&weight_of)); + } }); } From 2182abe649de3f209682501f9719e9f1b70e720a Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 15:03:20 +0200 Subject: [PATCH 42/82] bond_extra implicitly rebags --- frame/staking/src/benchmarking.rs | 11 +++++++- frame/staking/src/lib.rs | 34 ++++++++++++++---------- frame/staking/src/tests.rs | 43 ++++++++++++------------------- 3 files changed, 46 insertions(+), 42 deletions(-) diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 39338fb518e4d..208769ce1ccac 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -678,9 +678,18 @@ benchmarks! { let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; // update the stash account's value/weight + // + // note that we have to manually update the ledger; if we were to just call + // `Staking::::bond_extra`, then it would implicitly rebag. We want to separate that step + // so we can measure it in isolation. let other_free_balance = T::Currency::free_balance(&other_stash); T::Currency::make_free_balance_be(&stash, other_free_balance); - Staking::::bond_extra(RawOrigin::Signed(stash.clone()).into(), other_free_balance)?; + let controller = Staking::::bonded(&stash).ok_or("stash had no controller")?; + let mut ledger = Staking::::ledger(&controller).ok_or("controller had no ledger")?; + let extra = other_free_balance.checked_sub(&ledger.total).ok_or("balance did not increase")?; + ledger.total += extra; + ledger.active += extra; + Staking::::update_ledger(&controller, &ledger); // verify preconditions let weight_of = Staking::::weight_of_fn(); diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index c2c8dbbb36489..0abd609db6dfd 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1402,9 +1402,7 @@ pub mod pallet { Kicked(T::AccountId, T::AccountId), /// The election failed. No new era is planned. StakingElectionFailed, - /// Attempted to rebag an account. - /// - /// Contents: `who, from, to`. + /// Moved an account from one bag to another. \[who, from, to\]. Rebagged(T::AccountId, BagIdx, BagIdx), } @@ -1623,8 +1621,9 @@ pub mod pallet { // Last check: the new active amount of ledger must be more than ED. ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); - Self::deposit_event(Event::::Bonded(stash, extra)); + Self::deposit_event(Event::::Bonded(stash.clone(), extra)); Self::update_ledger(&controller, &ledger); + Self::do_rebag(&stash); } Ok(()) } @@ -2454,16 +2453,7 @@ pub mod pallet { stash: AccountIdOf, ) -> DispatchResult { ensure_signed(origin)?; - - // if no voter at that node, don't do anything. - // the caller just wasted the fee to call this. - if let Some((from, to)) = voter_bags::Node::::from_id(&stash).and_then(|node| { - let weight_of = Self::weight_of_fn(); - VoterList::update_position_for(node, weight_of) - }) { - Self::deposit_event(Event::::Rebagged(stash, from, to)); - }; - + Pallet::::do_rebag(&stash); Ok(()) } } @@ -3147,6 +3137,22 @@ impl Pallet { debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); } } + + /// Move a stash account from one bag to another, depositing an event on success. + /// + /// If the stash changed bags, returns `Some((from, to))`. + pub fn do_rebag(stash: &T::AccountId) -> Option<(BagIdx, BagIdx)> { + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { + let weight_of = Self::weight_of_fn(); + VoterList::update_position_for(node, weight_of) + }); + if let Some((from, to)) = maybe_movement { + Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); + }; + maybe_movement + } } impl diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index fa31e84ae47d3..e0e6b83d4b742 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3874,12 +3874,13 @@ fn on_finalize_weight_is_nonzero() { #[test] fn test_rebag() { use crate::{ - testing_utils::{create_funded_user, create_stash_controller}, + testing_utils::create_stash_controller, voter_bags::{Bag, Node}, }; use frame_system::RawOrigin; - fn make_validator(n: u32, balance_factor: u32) -> Result<(AccountIdOf, AccountIdOf), &'static str> { + /// Make a validator and return its stash + fn make_validator(n: u32, balance_factor: u32) -> Result, &'static str> { let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); // Bond the full value of the stash @@ -3894,30 +3895,18 @@ fn test_rebag() { ValidatorPrefs::default(), ).unwrap(); - Ok((stash, controller)) + Ok(stash) } ExtBuilder::default().build_and_execute(|| { // We want to have two validators: one, `stash`, is the one we will rebag. // The other, `other_stash`, exists only so that the destination bag is not empty. - let (stash, controller) = make_validator(0, 100).unwrap(); - let (other_stash, _) = make_validator(1, 300).unwrap(); - - // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. - let new_balance = ::Currency::free_balance(&other_stash); - ::Currency::make_free_balance_be(&stash, new_balance); - Staking::bond_extra( - RawOrigin::Signed(stash.clone()).into(), - new_balance, - ).unwrap(); + let stash = make_validator(0, 100).unwrap(); + let other_stash = make_validator(1, 300).unwrap(); // verify preconditions let weight_of = Staking::weight_of_fn(); let node = Node::::from_id(&stash).unwrap(); - assert!( - node.is_misplaced(&weight_of), - "rebagging only makes sense when a node is misplaced", - ); assert_eq!( { let origin_bag = Bag::::get(node.bag_idx).unwrap(); @@ -3937,18 +3926,18 @@ fn test_rebag() { "destination bag should not be empty", ); - // Any unrelated person can call `rebag`, as long as they sign and pay the tx fee. - let caller = create_funded_user::("caller", 3, 100); - // ensure it's distinct from the other accounts - assert_ne!(caller, stash); - assert_ne!(caller, other_stash); - assert_ne!(caller, controller); - - // call rebag - Pallet::::rebag(RawOrigin::Signed(caller).into(), stash.clone()).unwrap(); + // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. + // + // This implicitly calls rebag, so the user stays in the best bag they qualify for. + let new_balance = ::Currency::free_balance(&other_stash); + ::Currency::make_free_balance_be(&stash, new_balance); + Staking::bond_extra( + RawOrigin::Signed(stash.clone()).into(), + new_balance, + ).unwrap(); // node should no longer be misplaced - // note that we have to refresh the node + // note that we refresh the node, in case the storage value has changed let node = Node::::from_id(&stash).unwrap(); assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); }); From 472baa97099a5fa8a33cef11b838a616342df889 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 15:23:06 +0200 Subject: [PATCH 43/82] types at top; doc public type --- frame/staking/src/voter_bags/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 626ff010573f3..f768f056115b2 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -33,6 +33,9 @@ use frame_support::DefaultNoBound; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; +/// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. +pub type VoterOf = Voter>; + /// Index type for a bag. pub type BagIdx = u8; @@ -500,8 +503,6 @@ impl Voter { } } -pub type VoterOf = Voter>; - /// Type of voter. /// /// Similar to [`crate::StakerStatus`], but somewhat more limited. From f1e7fe7add9b8cc27007975c9e1efe3893cec545 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 15:01:40 +0200 Subject: [PATCH 44/82] start sketching out adjustable thresholds --- frame/staking/src/lib.rs | 53 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 0abd609db6dfd..34cdc0189d5d8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -960,6 +960,59 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; + + /// The list of thresholds separating the various voter bags. + /// + /// Voters are separated into unsorted bags according to their vote weight. This specifies + /// the thresholds separating the bags. A voter's bag is the largest bag for which the + /// voter's weight is less than or equal to its upper threshold. + /// + /// When voters are iterated, higher bags are iterated completely before lower bags. This + /// that iteration is _semi-sorted_: voters of higher weight tend to come before voters of + /// lower weight, but peer voters within a particular bag are sorted in insertion order. + /// + /// # Expressing the constant + /// + /// This constant must be sorted in strictly increasing order. Duplicate items are not + /// permitted. + /// + /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// specified within the bag. For any two threshold lists, if one ends with + /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists + /// will behave identically. + /// + /// # Calculation + /// + /// It is recommended to generate the set of thresholds in a geometric series, such that + /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * + /// constant_ratio).max(threshold[k] + 1)` for all `k`. + /// + /// Given the desire to compute `N` bags, the constant ratio can be computed with + /// `exp(ln(VoterBags::MAX) / N)`. + /// + /// # Examples + /// + /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, + /// and iteration is strictly in insertion order. + /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is equal to 2. + /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is approximately equal + /// to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will + /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. + /// + /// # Migration + /// + /// In the event that this list ever changes, the `VoterList` data structure will need to be + /// regenerated. + #[pallet::constant] + type VoterBagThresholds: Get<&'static [VoteWeight]>; + + /// A type sufficient to distinguish between all voter bags. + /// + /// For 256 bags or fewer, `u8` suffices. + type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen; } #[pallet::extra_constants] From 16e4774d05ca90ce843ab65d2b0004b06692de55 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 29 Jun 2021 16:06:36 +0200 Subject: [PATCH 45/82] add integrity test for voter bag threshold requirements --- frame/staking/src/lib.rs | 24 +++++++++++++++++------- frame/staking/src/mock.rs | 6 ++++++ frame/staking/src/voter_bags/mod.rs | 2 +- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 34cdc0189d5d8..b4d48fa398820 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -308,11 +308,11 @@ use frame_support::{ }; use pallet_session::historical; use sp_runtime::{ - Percent, Perbill, RuntimeDebug, DispatchError, + DispatchError, Perbill, Percent, RuntimeDebug, curve::PiecewiseLinear, traits::{ - Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, - AtLeast32BitUnsigned, Bounded, + AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, + StaticLookup, UniqueSaturatedInto, Zero, }, }; use sp_staking::{ @@ -1012,7 +1012,7 @@ pub mod pallet { /// A type sufficient to distinguish between all voter bags. /// /// For 256 bags or fewer, `u8` suffices. - type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen; + type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen + Bounded + UniqueSaturatedInto; } #[pallet::extra_constants] @@ -1550,14 +1550,24 @@ pub mod pallet { fn integrity_test() { sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| + sp_io::TestExternalities::new_empty().execute_with(|| { assert!( T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", T::SlashDeferDuration::get(), T::BondingDuration::get(), - ) - ); + ); + + assert!( + T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "Voter bag thresholds must strictly increase", + ); + + assert!( + T::BagIdx::max_value().saturated_into() >= T::VoterBagThresholds::get().len(), + "BagIdx must be sufficient to uniquely identify every bag", + ); + }); } } } diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index e0079cc3f375a..d9cc77653e986 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -243,6 +243,10 @@ impl onchain::Config for Test { type DataProvider = Staking; } +parameter_types! { + pub const VoterBagThresholds: &'static [VoteWeight] = &crate::voter_bags::thresholds::THRESHOLDS; +} + impl Config for Test { const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; @@ -263,6 +267,8 @@ impl Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = VoterBagThresholds; + type BagIdx = u8; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index f768f056115b2..29a42d35eb14a 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -21,7 +21,7 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -mod thresholds; +pub mod thresholds; use thresholds::THRESHOLDS; use crate::{ From 2a4ebf7f9fef531df96028716237e16ea6f248cd Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 30 Jun 2021 16:47:23 +0200 Subject: [PATCH 46/82] get rid of BagIdx This reorganizes bag storage such that bags are always referred to by their upper threshold. This in turn means that adding and removing bags is cheaper; you only need to migrate certain voters, not all of them. --- frame/staking/src/lib.rs | 36 +++---- frame/staking/src/voter_bags/mod.rs | 109 +++++++++++---------- frame/staking/src/voter_bags/thresholds.rs | 4 +- 3 files changed, 73 insertions(+), 76 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index b4d48fa398820..7f6b5402cff45 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -312,7 +312,7 @@ use sp_runtime::{ curve::PiecewiseLinear, traits::{ AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, - StaticLookup, UniqueSaturatedInto, Zero, + StaticLookup, Zero, }, }; use sp_staking::{ @@ -321,7 +321,7 @@ use sp_staking::{ }; use frame_system::{ensure_signed, ensure_root, pallet_prelude::*, offchain::SendTransactionTypes}; use frame_election_provider_support::{ElectionProvider, VoteWeight, Supports, data_provider}; -use voter_bags::{BagIdx, VoterList, VoterType}; +use voter_bags::{VoterList, VoterType}; pub use weights::WeightInfo; pub use pallet::*; @@ -1008,11 +1008,6 @@ pub mod pallet { /// regenerated. #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; - - /// A type sufficient to distinguish between all voter bags. - /// - /// For 256 bags or fewer, `u8` suffices. - type BagIdx: Copy + Eq + Ord + codec::Codec + MaxEncodedLen + Bounded + UniqueSaturatedInto; } #[pallet::extra_constants] @@ -1311,6 +1306,8 @@ pub mod pallet { // The next storage items collectively comprise the voter bags: a composite data structure // designed to allow efficient iteration of the top N voters by stake, mostly. See // `mod voter_bags` for details. + // + // In each of these items, voter bags are indexed by their upper weight threshold. /// How many voters are registered. #[pallet::storage] @@ -1321,21 +1318,25 @@ pub mod pallet { /// This may not be the appropriate bag for the voter's weight if they have been rewarded or /// slashed. #[pallet::storage] - pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::BagIdx>; + pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; - /// The head and tail of each bag of voters. + /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which + /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] pub(crate) type VoterBags = StorageMap< _, - Twox64Concat, voter_bags::BagIdx, + Twox64Concat, VoteWeight, voter_bags::Bag, >; - /// The nodes comprising each bag. + /// Voter nodes store links forward and back within their respective bags, the stash id, and + /// whether the voter is a validator or nominator. + /// + /// There is nothing in this map directly identifying to which bag a particular node belongs. + /// However, the `Node` data structure has helpers which can provide that information. #[pallet::storage] - pub(crate) type VoterNodes = StorageDoubleMap< + pub(crate) type VoterNodes = StorageMap< _, - Twox64Concat, voter_bags::BagIdx, Twox64Concat, AccountIdOf, voter_bags::Node, >; @@ -1456,7 +1457,7 @@ pub mod pallet { /// The election failed. No new era is planned. StakingElectionFailed, /// Moved an account from one bag to another. \[who, from, to\]. - Rebagged(T::AccountId, BagIdx, BagIdx), + Rebagged(T::AccountId, VoteWeight, VoteWeight), } #[pallet::error] @@ -1562,11 +1563,6 @@ pub mod pallet { T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), "Voter bag thresholds must strictly increase", ); - - assert!( - T::BagIdx::max_value().saturated_into() >= T::VoterBagThresholds::get().len(), - "BagIdx must be sufficient to uniquely identify every bag", - ); }); } } @@ -3204,7 +3200,7 @@ impl Pallet { /// Move a stash account from one bag to another, depositing an event on success. /// /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag(stash: &T::AccountId) -> Option<(BagIdx, BagIdx)> { + pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 29a42d35eb14a..11a54a6fe10a9 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -23,41 +23,37 @@ pub mod thresholds; -use thresholds::THRESHOLDS; use crate::{ AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; -use frame_support::DefaultNoBound; +use frame_support::{DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; -/// Index type for a bag. -pub type BagIdx = u8; - -/// How many bags there are -pub const N_BAGS: BagIdx = 200; - -/// Given a certain vote weight, which bag should this voter contain? +/// Given a certain vote weight, which bag should contain this voter? +/// +/// Bags are identified by their upper threshold; the value returned by this function is guaranteed +/// to be a member of `T::VoterBagThresholds`. /// -/// Bags are separated by fixed thresholds. To the extent possible, each threshold is a constant -/// small multiple of the one before it. That ratio is [`thresholds::CONSTANT_RATIO`]. The exception -/// are the smallest bags, which are each at least 1 greater than the previous, and the largest bag, -/// which is defined as `u64::MAX`. +/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, +/// because in the event that bags are inserted or deleted, the number of affected voters which need +/// to be migrated is smaller. /// -/// Bags are arranged such that `bags[0]` is the largest bag, and `bags[N_BAGS-1]` is the smallest. -fn notional_bag_for(weight: VoteWeight) -> BagIdx { - let raw_bag = - THRESHOLDS.partition_point(|&threshold| weight > threshold) as BagIdx; - N_BAGS - 1 - raw_bag +/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this +/// function behaves as if it does. +fn notional_bag_for(weight: VoteWeight) -> VoteWeight { + let thresholds = T::VoterBagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| weight > threshold); + thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) } -/// Find the actual bag containing the current voter. -fn current_bag_for(id: &AccountIdOf) -> Option { +/// Find the upper threshold of the actual bag containing the current voter. +fn current_bag_for(id: &AccountIdOf) -> Option { VoterBagFor::::try_get(id).ok() } @@ -110,10 +106,13 @@ impl VoterList { /// Iterate over all nodes in all bags in the voter list. /// - /// Note that this exhaustively attempts to try all possible bag indices. Full iteration can be - /// expensive; it's recommended to limit the number of items with `.take(n)`. + /// Full iteration can be expensive; it's recommended to limit the number of items with `.take(n)`. pub fn iter() -> impl Iterator> { - (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) + T::VoterBagThresholds::get() + .iter() + .copied() + .filter_map(Bag::get) + .flat_map(|bag| bag.iter()) } /// Insert a new voter into the appropriate bag in the voter list. @@ -147,7 +146,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); - let bag = notional_bag_for(weight); + let bag = notional_bag_for::(weight); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -180,11 +179,11 @@ impl VoterList { count += 1; // clear the bag head/tail pointers as necessary - let bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); + let bag = bags.entry(node.bag_upper).or_insert_with(|| Bag::::get_or_make(node.bag_upper)); bag.remove_node(&node); // now get rid of the node itself - crate::VoterNodes::::remove(node.bag_idx, voter_id); + crate::VoterNodes::::remove(voter_id); crate::VoterBagFor::::remove(voter_id); } @@ -208,12 +207,12 @@ impl VoterList { pub fn update_position_for( mut node: Node, weight_of: impl Fn(&AccountIdOf) -> VoteWeight, - ) -> Option<(BagIdx, BagIdx)> { + ) -> Option<(VoteWeight, VoteWeight)> { node.is_misplaced(&weight_of).then(move || { - let old_idx = node.bag_idx; + let old_idx = node.bag_upper; // clear the old bag head/tail pointers as necessary - if let Some(mut bag) = Bag::::get(node.bag_idx) { + if let Some(mut bag) = Bag::::get(node.bag_upper) { bag.remove_node(&node); bag.put(); } else { @@ -226,9 +225,9 @@ impl VoterList { } // put the voter into the appropriate new bag - let new_idx = notional_bag_for(weight_of(&node.voter.id)); - node.bag_idx = new_idx; - let mut bag = Bag::::get_or_make(node.bag_idx); + let new_idx = notional_bag_for::(weight_of(&node.voter.id)); + node.bag_upper = new_idx; + let mut bag = Bag::::get_or_make(node.bag_upper); bag.insert_node(node); bag.put(); @@ -250,36 +249,40 @@ pub struct Bag { tail: Option>, #[codec(skip)] - bag_idx: BagIdx, + bag_upper: VoteWeight, } impl Bag { - /// Get a bag by idx. - pub fn get(bag_idx: BagIdx) -> Option> { - crate::VoterBags::::try_get(bag_idx).ok().map(|mut bag| { - bag.bag_idx = bag_idx; + /// Get a bag by its upper vote weight. + pub fn get(bag_upper: VoteWeight) -> Option> { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; bag }) } - /// Get a bag by idx or make it, appropriately initialized. - pub fn get_or_make(bag_idx: BagIdx) -> Bag { - Self::get(bag_idx).unwrap_or(Bag { bag_idx, ..Default::default() }) + /// Get a bag by its upper vote weight or make it, appropriately initialized. + pub fn get_or_make(bag_upper: VoteWeight) -> Bag { + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } /// Put the bag back into storage. pub fn put(self) { - crate::VoterBags::::insert(self.bag_idx, self); + crate::VoterBags::::insert(self.bag_upper, self); } /// Get the head node in this bag. pub fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) } /// Get the tail node in this bag. pub fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(self.bag_idx, id)) + self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) } /// Iterate over the nodes in this bag. @@ -299,7 +302,7 @@ impl Bag { voter, prev: None, next: None, - bag_idx: self.bag_idx, + bag_upper: self.bag_upper, }); } @@ -329,7 +332,7 @@ impl Bag { } self.tail = Some(id.clone()); - crate::VoterBagFor::::insert(id, self.bag_idx); + crate::VoterBagFor::::insert(id, self.bag_upper); } /// Remove a voter node from this bag. @@ -362,14 +365,14 @@ pub struct Node { /// The bag index is not stored in storage, but injected during all fetch operations. #[codec(skip)] - pub(crate) bag_idx: BagIdx, + pub(crate) bag_upper: VoteWeight, } impl Node { /// Get a node by bag idx and account id. - pub fn get(bag_idx: BagIdx, account_id: &AccountIdOf) -> Option> { - crate::VoterNodes::::try_get(&bag_idx, account_id).ok().map(|mut node| { - node.bag_idx = bag_idx; + pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { + crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { + node.bag_upper = bag_upper; node }) } @@ -385,12 +388,12 @@ impl Node { /// Get a node by account id, assuming it's in the same bag as this node. pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { - Self::get(self.bag_idx, account_id) + Self::get(self.bag_upper, account_id) } /// Put the node back into storage. pub fn put(self) { - crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); + crate::VoterNodes::::insert(self.voter.id.clone(), self); } /// Get the previous node in the bag. @@ -448,7 +451,7 @@ impl Node { /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx + notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper } /// Update the voter type associated with a particular node by id. @@ -470,7 +473,7 @@ impl Node { /// /// This is a helper intended only for benchmarking and should not be used in production. #[cfg(any(test, feature = "runtime-benchmarks"))] - pub fn proper_bag_for(&self) -> BagIdx { + pub fn proper_bag_for(&self) -> VoteWeight { let weight_of = crate::Pallet::::weight_of_fn(); let current_weight = weight_of(&self.voter.id); notional_bag_for(current_weight) diff --git a/frame/staking/src/voter_bags/thresholds.rs b/frame/staking/src/voter_bags/thresholds.rs index 710403d75e64f..37b889405ac19 100644 --- a/frame/staking/src/voter_bags/thresholds.rs +++ b/frame/staking/src/voter_bags/thresholds.rs @@ -17,15 +17,13 @@ //! Generated voter bag thresholds. -use super::N_BAGS; - /// Ratio between adjacent bags; #[cfg(any(test, feature = "std"))] #[allow(unused)] pub const CONSTANT_RATIO: f64 = 1.2483305489016119; /// Upper thresholds for each bag. -pub const THRESHOLDS: [u64; N_BAGS as usize] = [ +pub const THRESHOLDS: [u64; 200] = [ 1, 2, 3, From cc6d0dff5a78b9eb5af7c74ca2c974fde5fd1020 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 1 Jul 2021 11:57:33 +0200 Subject: [PATCH 47/82] implement migration logic for when the threshold list changes --- frame/staking/src/voter_bags/mod.rs | 94 ++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index 11a54a6fe10a9..a3283bbdb0262 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -30,7 +30,7 @@ use crate::{ use codec::{Encode, Decode}; use frame_support::{DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; -use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; +use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, marker::PhantomData}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; @@ -234,6 +234,94 @@ impl VoterList { (old_idx, new_idx) }) } + + /// Migrate the voter list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::VoterBagThresholds` has already been updated. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - No voter is changed unless required to by the difference between the old threshold list + /// and the new. + /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// new threshold set. + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + // we can't check all preconditions, but we can check one + debug_assert!( + crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); + + let mut affected_accounts = BTreeSet::new(); + let mut affected_old_bags = BTreeSet::new(); + + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_set.difference(&old_set).copied() { + let affected_bag = notional_bag_for::(inserted_bag); + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // a removed bag means that all members of that bag must be rebagged + for removed_bag in old_set.difference(&new_set).copied() { + if !affected_old_bags.insert(removed_bag) { + continue + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // migrate the + let weight_of = Pallet::::weight_of_fn(); + Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); + let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in old_set.difference(&new_set).copied() { + debug_assert!( + !VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + "no voter should be present in a removed bag", + ); + crate::VoterBags::::remove(removed_bag); + } + + debug_assert!( + { + let thresholds = T::VoterBagThresholds::get(); + crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + }, + "all `bag_upper` in storage must be members of the new thresholds", + ); + + num_affected + } } /// A Bag is a doubly-linked list of voters. @@ -481,7 +569,7 @@ impl Node { } /// Fundamental information about a voter. -#[derive(Clone, Encode, Decode)] +#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(feature = "std", derive(Debug))] pub struct Voter { /// Account Id of this voter @@ -509,7 +597,7 @@ impl Voter { /// Type of voter. /// /// Similar to [`crate::StakerStatus`], but somewhat more limited. -#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] #[cfg_attr(feature = "std", derive(Debug))] pub enum VoterType { Validator, From 88195fe0f885fcf9b58d3f85c649d4e72ab456f2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 1 Jul 2021 14:27:25 +0200 Subject: [PATCH 48/82] start sketching out threshold proc macros --- Cargo.lock | 10 +++ Cargo.toml | 1 + frame/staking/bag-thresholds/Cargo.toml | 18 +++++ frame/staking/bag-thresholds/src/lib.rs | 92 +++++++++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 frame/staking/bag-thresholds/Cargo.toml create mode 100644 frame/staking/bag-thresholds/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index cb45b18013992..6323931a96459 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5530,6 +5530,16 @@ dependencies = [ "sp-arithmetic", ] +[[package]] +name = "pallet-staking-voter-bag-thresholds" +version = "3.0.0" +dependencies = [ + "proc-macro-crate 1.0.0", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pallet-sudo" version = "3.0.0" diff --git a/Cargo.toml b/Cargo.toml index f7552f0bbbc48..bec00c77bcbe5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ members = [ "frame/session/benchmarking", "frame/society", "frame/staking", + "frame/staking/bag-thresholds", "frame/staking/reward-curve", "frame/staking/reward-fn", "frame/sudo", diff --git a/frame/staking/bag-thresholds/Cargo.toml b/frame/staking/bag-thresholds/Cargo.toml new file mode 100644 index 0000000000000..44454eb4e28e6 --- /dev/null +++ b/frame/staking/bag-thresholds/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pallet-staking-voter-bag-thresholds" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Voter Bag Thresholds for FRAME Staking Pallet" + +[lib] +proc-macro = true + +[dependencies] +proc-macro2 = "1.0.6" +proc-macro-crate = "1.0.0" +quote = "1.0.3" +syn = { version = "1.0.58", features = ["full", "visit"] } diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs new file mode 100644 index 0000000000000..5d87658e875c8 --- /dev/null +++ b/frame/staking/bag-thresholds/src/lib.rs @@ -0,0 +1,92 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Proc macros to generate voter bag thresholds and associated items. + +/// Calculate an appropriate constant ratio between thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn make_ratio(n: usize, bounds: impl std::ops::RangeBounds) -> f64; +/// ``` +/// +/// # Example: +/// +/// ``` +/// # use pallet_staking_voter_bag_thresholds::make_ratio; +/// /// Constant ratio between bag items. Approx. `1.248`. +/// const CONSTANT_RATIO: f64 = make_ratio!(200, ..); +/// ``` +#[proc_macro] +pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + todo!() +} + +/// Make a constant array of threshold values suitable for use as voter bag thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn make_thresholds(n: usize, bounds: impl std::ops::RangeBounds) -> [VoteWeight; n]; +/// ``` +/// +/// The output has these properties: +/// +/// - Its length is `n`. +/// - Its first item respects `bounds.start_bound()`. +/// - Its last item respects `bounds.end_bound()`. +/// - There exists a constant ratio (see [`make_ratio`]) called _ratio_. +/// +/// For all _k_, `output[k + 1] == (output[k] * ratio).round().min(output[k] + 1)`. +/// +/// # Example: +/// +/// ``` +/// # use pallet_staking_voter_bag_thresholds::make_thresholds; +/// const THRESHOLDS: &[u64] = &make_thresholds!(200, ..); +/// ``` +#[proc_macro] +pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + todo!() +} + +/// Make a constant array of threshold values suitable for use as voter bag thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn edit_thresholds( +/// thresholds: [VoteWeight; Old], +/// inserting: impl IntoIterator, +/// removing: impl IntoIterator, +/// ) -> [VoteWeight; New]; +/// ``` +/// +/// It is intended to be used to simply edit a thresholds list, inserting some items and removing +/// others, without needing to express the entire list. +/// +/// # Example: +/// +/// ``` +/// # use pallet_staking_voter_bag_thresholds::{edit_thresholds, make_thresholds}; +/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, ..), [12345, 54321], []); +/// ``` +#[proc_macro] +pub fn edit_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + todo!() +} From 70848eacdae04dd9413eb829814b1c136a092089 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Thu, 1 Jul 2021 15:48:55 +0200 Subject: [PATCH 49/82] further refine macro signatures --- frame/staking/bag-thresholds/src/lib.rs | 52 +++++++++++++++++++------ 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs index 5d87658e875c8..aec0ce4284a05 100644 --- a/frame/staking/bag-thresholds/src/lib.rs +++ b/frame/staking/bag-thresholds/src/lib.rs @@ -22,16 +22,27 @@ /// This macro can be thought of as a function with signature /// /// ```ignore -/// pub const fn make_ratio(n: usize, bounds: impl std::ops::RangeBounds) -> f64; +/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; /// ``` /// +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a +/// `::MAX` attribute available. +/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, +/// this will be the result of some calculation involving the existential deposit for a chain's +/// balance type. +/// /// # Example: /// /// ``` /// # use pallet_staking_voter_bag_thresholds::make_ratio; /// /// Constant ratio between bag items. Approx. `1.248`. -/// const CONSTANT_RATIO: f64 = make_ratio!(200, ..); +/// const CONSTANT_RATIO: f64 = make_ratio!(200, u64, 0); /// ``` +/// +/// # Calculation +/// +/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). #[proc_macro] pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { todo!() @@ -42,23 +53,37 @@ pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// This macro can be thought of as a function with signature /// /// ```ignore -/// pub const fn make_thresholds(n: usize, bounds: impl std::ops::RangeBounds) -> [VoteWeight; n]; +/// pub const fn make_thresholds(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> [VoteWeight; n]; /// ``` /// +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a +/// `::MAX` attribute available. +/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, +/// this will be the result of some calculation involving the existential deposit for a chain's +/// balance type. +/// /// The output has these properties: /// /// - Its length is `n`. -/// - Its first item respects `bounds.start_bound()`. -/// - Its last item respects `bounds.end_bound()`. +/// - Its first item is greater than or equal to `existential_weight`. +/// - Its last item is equal to `VoteWeight::MAX`. /// - There exists a constant ratio (see [`make_ratio`]) called _ratio_. /// -/// For all _k_, `output[k + 1] == (output[k] * ratio).round().min(output[k] + 1)`. +/// For all _k_ in `0..(n-1)`, `output[k + 1] == (output[k] * ratio).round()`. +/// +/// However, there are two exceptions to the ratio rule: +/// +/// - As thresholds may not duplicate, if `(output[k] * ratio).round() == output[k]`, then `output[k +/// + 1] == output[k] + 1`. +/// - Due to the previous exception in combination with the requirement that the final item is equal +/// to `VoteWeight::MAX`, the ratio of the final item may diverge from the common ratio. /// /// # Example: /// /// ``` /// # use pallet_staking_voter_bag_thresholds::make_thresholds; -/// const THRESHOLDS: &[u64] = &make_thresholds!(200, ..); +/// const THRESHOLDS: [u64; 200] = &make_thresholds!(200, u64, 0); /// ``` #[proc_macro] pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -70,10 +95,15 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// This macro can be thought of as a function with signature /// /// ```ignore -/// pub const fn edit_thresholds( +/// pub const fn edit_thresholds< +/// const Old: usize, +/// const Inserting: usize, +/// const Removing: usize, +/// const New: usize, +/// >( /// thresholds: [VoteWeight; Old], -/// inserting: impl IntoIterator, -/// removing: impl IntoIterator, +/// inserting: [VoteWeight; Inserting], +/// removing: [VoteWeight; Removing], /// ) -> [VoteWeight; New]; /// ``` /// @@ -84,7 +114,7 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// /// ``` /// # use pallet_staking_voter_bag_thresholds::{edit_thresholds, make_thresholds}; -/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, ..), [12345, 54321], []); +/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, u64, 0), [12345, 54321], []); /// ``` #[proc_macro] pub fn edit_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { From b6303213abaa160f2cce8cbd23ad0beed114ecc2 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 11:36:29 +0200 Subject: [PATCH 50/82] WIP: implement make_ratio macro --- frame/staking/bag-thresholds/src/common.rs | 55 +++++++++++++++++++ frame/staking/bag-thresholds/src/lib.rs | 28 +++++++--- .../bag-thresholds/src/make_ratio_impl.rs | 51 +++++++++++++++++ frame/staking/bag-thresholds/tests/tests.rs | 30 ++++++++++ 4 files changed, 156 insertions(+), 8 deletions(-) create mode 100644 frame/staking/bag-thresholds/src/common.rs create mode 100644 frame/staking/bag-thresholds/src/make_ratio_impl.rs create mode 100644 frame/staking/bag-thresholds/tests/tests.rs diff --git a/frame/staking/bag-thresholds/src/common.rs b/frame/staking/bag-thresholds/src/common.rs new file mode 100644 index 0000000000000..4aa5fc4892066 --- /dev/null +++ b/frame/staking/bag-thresholds/src/common.rs @@ -0,0 +1,55 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Features common between the other modules of this crate. + +use syn::{Expr, Result, Token, Type, parse::{Parse, ParseStream}}; + +/// Parse function-like input parameters +/// +/// ```ignore +/// fn my_fn(n: usize, VoteWeight: Type, existential_weight: VoteWeight); +/// ``` +/// +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any +/// expression which is evaluable in a const context. +/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a +/// `::MAX` attribute available. +/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, +/// this will be the result of some calculation involving the existential deposit for a chain's +/// balance type. It may be any expression which is evaluable in a const context. +pub struct ThresholdParams { + pub n: Expr, + pub comma1: Token![,], + pub vote_weight: Type, + pub comma2: Token![,], + pub existential_weight: Expr, + pub comma3: Option, +} + +impl Parse for ThresholdParams { + fn parse(input: ParseStream) -> Result { + Ok(Self { + n: input.parse()?, + comma1: input.parse()?, + vote_weight: input.parse()?, + comma2: input.parse()?, + existential_weight: input.parse()?, + comma3: input.parse()?, + }) + } +} diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs index aec0ce4284a05..6ae7c68e42516 100644 --- a/frame/staking/bag-thresholds/src/lib.rs +++ b/frame/staking/bag-thresholds/src/lib.rs @@ -17,6 +17,11 @@ //! Proc macros to generate voter bag thresholds and associated items. +use proc_macro::TokenStream; + +mod common; +mod make_ratio_impl; + /// Calculate an appropriate constant ratio between thresholds. /// /// This macro can be thought of as a function with signature @@ -25,12 +30,13 @@ /// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; /// ``` /// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any +/// expression which is evaluable in a const context. /// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a /// `::MAX` attribute available. /// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, /// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. +/// balance type. It may be any expression which is evaluable in a const context. /// /// # Example: /// @@ -44,8 +50,8 @@ /// /// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). #[proc_macro] -pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { - todo!() +pub fn make_ratio(input: TokenStream) -> TokenStream { + make_ratio_impl::make_ratio(input) } /// Make a constant array of threshold values suitable for use as voter bag thresholds. @@ -56,12 +62,13 @@ pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// pub const fn make_thresholds(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> [VoteWeight; n]; /// ``` /// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. +/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any +/// expression which is evaluable in a const context. /// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a /// `::MAX` attribute available. /// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, /// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. +/// balance type. It may be any expression which is evaluable in a const context. /// /// The output has these properties: /// @@ -86,7 +93,7 @@ pub fn make_ratio(input: proc_macro::TokenStream) -> proc_macro::TokenStream { /// const THRESHOLDS: [u64; 200] = &make_thresholds!(200, u64, 0); /// ``` #[proc_macro] -pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +pub fn make_thresholds(input: TokenStream) -> TokenStream { todo!() } @@ -110,6 +117,11 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// It is intended to be used to simply edit a thresholds list, inserting some items and removing /// others, without needing to express the entire list. /// +/// Note that due to macro limitations, `inserting` and `removing` must be array literals. +/// +/// Note that `removing` overrides `inserting`: if a value appears in both lists, it does not appear +/// in the output. +/// /// # Example: /// /// ``` @@ -117,6 +129,6 @@ pub fn make_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStrea /// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, u64, 0), [12345, 54321], []); /// ``` #[proc_macro] -pub fn edit_thresholds(input: proc_macro::TokenStream) -> proc_macro::TokenStream { +pub fn edit_thresholds(input: TokenStream) -> TokenStream { todo!() } diff --git a/frame/staking/bag-thresholds/src/make_ratio_impl.rs b/frame/staking/bag-thresholds/src/make_ratio_impl.rs new file mode 100644 index 0000000000000..29812e7efec63 --- /dev/null +++ b/frame/staking/bag-thresholds/src/make_ratio_impl.rs @@ -0,0 +1,51 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation for the `make_ratio` proc macro. + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, parse_quote}; + +use crate::common::ThresholdParams; + +/// Calculate an appropriate constant ratio between thresholds. +/// +/// This macro can be thought of as a function with signature +/// +/// ```ignore +/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; +/// ``` +/// +/// # Calculation +/// +/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). +pub fn make_ratio(input: TokenStream) -> TokenStream { + let ThresholdParams{ + n, + vote_weight, + existential_weight, + .. + } = parse_macro_input!(input as ThresholdParams); + + let n = parse_quote!(#n) as f64; + let diff_weight = parse_quote!(#vote_weight::MAX - #existential_weight) as f64; + + let ratio = (diff_weight.ln() / n).exp(); + + quote!(#ratio).into() +} diff --git a/frame/staking/bag-thresholds/tests/tests.rs b/frame/staking/bag-thresholds/tests/tests.rs new file mode 100644 index 0000000000000..7618457476719 --- /dev/null +++ b/frame/staking/bag-thresholds/tests/tests.rs @@ -0,0 +1,30 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod make_ratio { + use pallet_staking_voter_bag_thresholds::make_ratio; + + #[test] + fn u64_200_0() { + assert_eq!(make_ratio!(200, u64, 0), 1.2483305489016119); + } + + #[test] + fn u64_64_0() { + assert_eq!(make_ratio!(64, u64, 0), 2.0); + } +} From 8ce3bc0e78b5290e8758eac0d8717976ecd2dca9 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 13:47:23 +0200 Subject: [PATCH 51/82] start rethinking the process of producing threshold lists The macro approach seems to be a non-starter; that only really works if we're throwing around numeric literals everywhere, and that's just not nice in this case. Instead, let's write helper functions and make it really easy to generate the tables in separate, permanent files, which humans can then edit. --- Cargo.lock | 37 +++-- Cargo.toml | 1 - frame/staking/Cargo.toml | 7 +- frame/staking/bag-thresholds/Cargo.toml | 18 --- frame/staking/bag-thresholds/src/common.rs | 55 ------- frame/staking/bag-thresholds/src/lib.rs | 134 ------------------ .../bag-thresholds/src/make_ratio_impl.rs | 51 ------- frame/staking/bag-thresholds/tests/tests.rs | 30 ---- frame/staking/src/bin/make_bags.rs | 4 +- frame/staking/src/voter_bags/mod.rs | 36 +++++ 10 files changed, 70 insertions(+), 303 deletions(-) delete mode 100644 frame/staking/bag-thresholds/Cargo.toml delete mode 100644 frame/staking/bag-thresholds/src/common.rs delete mode 100644 frame/staking/bag-thresholds/src/lib.rs delete mode 100644 frame/staking/bag-thresholds/src/make_ratio_impl.rs delete mode 100644 frame/staking/bag-thresholds/tests/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 6323931a96459..c391dee935205 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2261,6 +2261,19 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "git2" +version = "0.13.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url 2.2.1", +] + [[package]] name = "glob" version = "0.3.0" @@ -3140,6 +3153,18 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" +[[package]] +name = "libgit2-sys" +version = "0.12.21+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.5.2" @@ -3635,6 +3660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] @@ -5486,6 +5512,7 @@ dependencies = [ "frame-election-provider-support", "frame-support", "frame-system", + "git2", "hex", "log", "pallet-authorship", @@ -5530,16 +5557,6 @@ dependencies = [ "sp-arithmetic", ] -[[package]] -name = "pallet-staking-voter-bag-thresholds" -version = "3.0.0" -dependencies = [ - "proc-macro-crate 1.0.0", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pallet-sudo" version = "3.0.0" diff --git a/Cargo.toml b/Cargo.toml index bec00c77bcbe5..f7552f0bbbc48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,7 +108,6 @@ members = [ "frame/session/benchmarking", "frame/society", "frame/staking", - "frame/staking/bag-thresholds", "frame/staking/reward-curve", "frame/staking/reward-fn", "frame/sudo", diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 3e080f2537014..1988e804c54e0 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -33,6 +33,9 @@ paste = "1.0" frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true } rand_chacha = { version = "0.2", default-features = false, optional = true } +# Optional imports for making voter bags lists +git2 = { version = "0.13.20", default-features = false, optional = true } + [dev-dependencies] sp-storage = { version = "3.0.0", path = "../../primitives/storage" } sp-tracing = { version = "3.0.0", path = "../../primitives/tracing" } @@ -71,8 +74,8 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] -make-bags = [] +make-bags = ["std", "git2"] [[bin]] name = "make_bags" -required-features = ["make-bags", "std"] +required-features = ["make-bags"] diff --git a/frame/staking/bag-thresholds/Cargo.toml b/frame/staking/bag-thresholds/Cargo.toml deleted file mode 100644 index 44454eb4e28e6..0000000000000 --- a/frame/staking/bag-thresholds/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "pallet-staking-voter-bag-thresholds" -version = "3.0.0" -authors = ["Parity Technologies "] -edition = "2018" -license = "Apache-2.0" -homepage = "https://substrate.dev" -repository = "https://github.com/paritytech/substrate/" -description = "Voter Bag Thresholds for FRAME Staking Pallet" - -[lib] -proc-macro = true - -[dependencies] -proc-macro2 = "1.0.6" -proc-macro-crate = "1.0.0" -quote = "1.0.3" -syn = { version = "1.0.58", features = ["full", "visit"] } diff --git a/frame/staking/bag-thresholds/src/common.rs b/frame/staking/bag-thresholds/src/common.rs deleted file mode 100644 index 4aa5fc4892066..0000000000000 --- a/frame/staking/bag-thresholds/src/common.rs +++ /dev/null @@ -1,55 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Features common between the other modules of this crate. - -use syn::{Expr, Result, Token, Type, parse::{Parse, ParseStream}}; - -/// Parse function-like input parameters -/// -/// ```ignore -/// fn my_fn(n: usize, VoteWeight: Type, existential_weight: VoteWeight); -/// ``` -/// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any -/// expression which is evaluable in a const context. -/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a -/// `::MAX` attribute available. -/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, -/// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. It may be any expression which is evaluable in a const context. -pub struct ThresholdParams { - pub n: Expr, - pub comma1: Token![,], - pub vote_weight: Type, - pub comma2: Token![,], - pub existential_weight: Expr, - pub comma3: Option, -} - -impl Parse for ThresholdParams { - fn parse(input: ParseStream) -> Result { - Ok(Self { - n: input.parse()?, - comma1: input.parse()?, - vote_weight: input.parse()?, - comma2: input.parse()?, - existential_weight: input.parse()?, - comma3: input.parse()?, - }) - } -} diff --git a/frame/staking/bag-thresholds/src/lib.rs b/frame/staking/bag-thresholds/src/lib.rs deleted file mode 100644 index 6ae7c68e42516..0000000000000 --- a/frame/staking/bag-thresholds/src/lib.rs +++ /dev/null @@ -1,134 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Proc macros to generate voter bag thresholds and associated items. - -use proc_macro::TokenStream; - -mod common; -mod make_ratio_impl; - -/// Calculate an appropriate constant ratio between thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; -/// ``` -/// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any -/// expression which is evaluable in a const context. -/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a -/// `::MAX` attribute available. -/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, -/// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. It may be any expression which is evaluable in a const context. -/// -/// # Example: -/// -/// ``` -/// # use pallet_staking_voter_bag_thresholds::make_ratio; -/// /// Constant ratio between bag items. Approx. `1.248`. -/// const CONSTANT_RATIO: f64 = make_ratio!(200, u64, 0); -/// ``` -/// -/// # Calculation -/// -/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). -#[proc_macro] -pub fn make_ratio(input: TokenStream) -> TokenStream { - make_ratio_impl::make_ratio(input) -} - -/// Make a constant array of threshold values suitable for use as voter bag thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn make_thresholds(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> [VoteWeight; n]; -/// ``` -/// -/// - The argument `n` is how many divisions we're partitioning `VoteWeight` into. It may be any -/// expression which is evaluable in a const context. -/// - `VoteWeight` is the type of the vote weight, and should typically be a typedef. It must have a -/// `::MAX` attribute available. -/// - `existential_weight` is the weight below which it's not worth examining a voter. Typically, -/// this will be the result of some calculation involving the existential deposit for a chain's -/// balance type. It may be any expression which is evaluable in a const context. -/// -/// The output has these properties: -/// -/// - Its length is `n`. -/// - Its first item is greater than or equal to `existential_weight`. -/// - Its last item is equal to `VoteWeight::MAX`. -/// - There exists a constant ratio (see [`make_ratio`]) called _ratio_. -/// -/// For all _k_ in `0..(n-1)`, `output[k + 1] == (output[k] * ratio).round()`. -/// -/// However, there are two exceptions to the ratio rule: -/// -/// - As thresholds may not duplicate, if `(output[k] * ratio).round() == output[k]`, then `output[k -/// + 1] == output[k] + 1`. -/// - Due to the previous exception in combination with the requirement that the final item is equal -/// to `VoteWeight::MAX`, the ratio of the final item may diverge from the common ratio. -/// -/// # Example: -/// -/// ``` -/// # use pallet_staking_voter_bag_thresholds::make_thresholds; -/// const THRESHOLDS: [u64; 200] = &make_thresholds!(200, u64, 0); -/// ``` -#[proc_macro] -pub fn make_thresholds(input: TokenStream) -> TokenStream { - todo!() -} - -/// Make a constant array of threshold values suitable for use as voter bag thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn edit_thresholds< -/// const Old: usize, -/// const Inserting: usize, -/// const Removing: usize, -/// const New: usize, -/// >( -/// thresholds: [VoteWeight; Old], -/// inserting: [VoteWeight; Inserting], -/// removing: [VoteWeight; Removing], -/// ) -> [VoteWeight; New]; -/// ``` -/// -/// It is intended to be used to simply edit a thresholds list, inserting some items and removing -/// others, without needing to express the entire list. -/// -/// Note that due to macro limitations, `inserting` and `removing` must be array literals. -/// -/// Note that `removing` overrides `inserting`: if a value appears in both lists, it does not appear -/// in the output. -/// -/// # Example: -/// -/// ``` -/// # use pallet_staking_voter_bag_thresholds::{edit_thresholds, make_thresholds}; -/// const THRESHOLDS: &[u64] = &edit_thresholds!(make_thresholds!(200, u64, 0), [12345, 54321], []); -/// ``` -#[proc_macro] -pub fn edit_thresholds(input: TokenStream) -> TokenStream { - todo!() -} diff --git a/frame/staking/bag-thresholds/src/make_ratio_impl.rs b/frame/staking/bag-thresholds/src/make_ratio_impl.rs deleted file mode 100644 index 29812e7efec63..0000000000000 --- a/frame/staking/bag-thresholds/src/make_ratio_impl.rs +++ /dev/null @@ -1,51 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Implementation for the `make_ratio` proc macro. - -use proc_macro::TokenStream; -use quote::quote; -use syn::{parse_macro_input, parse_quote}; - -use crate::common::ThresholdParams; - -/// Calculate an appropriate constant ratio between thresholds. -/// -/// This macro can be thought of as a function with signature -/// -/// ```ignore -/// pub const fn make_ratio(n: usize, VoteWeight: Type, existential_weight: VoteWeight) -> f64; -/// ``` -/// -/// # Calculation -/// -/// The constant ratio is calculated per `exp(ln(VoteWeight::MAX - existential_weight) / n). -pub fn make_ratio(input: TokenStream) -> TokenStream { - let ThresholdParams{ - n, - vote_weight, - existential_weight, - .. - } = parse_macro_input!(input as ThresholdParams); - - let n = parse_quote!(#n) as f64; - let diff_weight = parse_quote!(#vote_weight::MAX - #existential_weight) as f64; - - let ratio = (diff_weight.ln() / n).exp(); - - quote!(#ratio).into() -} diff --git a/frame/staking/bag-thresholds/tests/tests.rs b/frame/staking/bag-thresholds/tests/tests.rs deleted file mode 100644 index 7618457476719..0000000000000 --- a/frame/staking/bag-thresholds/tests/tests.rs +++ /dev/null @@ -1,30 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod make_ratio { - use pallet_staking_voter_bag_thresholds::make_ratio; - - #[test] - fn u64_200_0() { - assert_eq!(make_ratio!(200, u64, 0), 1.2483305489016119); - } - - #[test] - fn u64_64_0() { - assert_eq!(make_ratio!(64, u64, 0), 2.0); - } -} diff --git a/frame/staking/src/bin/make_bags.rs b/frame/staking/src/bin/make_bags.rs index 664c7b5470ba0..a5773c59522c3 100644 --- a/frame/staking/src/bin/make_bags.rs +++ b/frame/staking/src/bin/make_bags.rs @@ -17,7 +17,7 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::voter_bags::N_BAGS; +const N_BAGS: usize = 200; fn main() { let ratio = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); @@ -27,7 +27,7 @@ fn main() { while thresholds.len() < N_BAGS as usize { let prev_item: u64 = thresholds.last().copied().unwrap_or_default(); - let item = (prev_item as f64 * ratio).max(prev_item as f64 + 1.0); + let item = (prev_item as f64 * ratio).round().max(prev_item as f64 + 1.0); thresholds.push(item as u64); } diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index a3283bbdb0262..dd5869afe007a 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -603,3 +603,39 @@ pub enum VoterType { Validator, Nominator, } + +/// Support code to ease the process of generating voter bags. +#[cfg(feature = "make-bags")] +pub mod make_bags { + use crate::{AccountIdOf, Config}; + use frame_election_provider_support::VoteWeight; + use frame_support::traits::{Currency, CurrencyToVote}; + use std::path::{Path, PathBuf}; + + /// Return the path to a header file used in this repository if is exists. + /// + /// Just searches the git working directory root for files matching certain patterns; it's + /// pretty naive. + fn path_to_header_file() -> Option { + let repo = git2::Repository::open_from_env().ok()?; + let workdir = repo.workdir()?; + for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER"] { + let path = workdir.join(file_name); + if path.exists() { + return Some(path); + } + } + None + } + + /// Compute the existential weight for the specified configuration. + /// + /// Note that this value depends on the current issuance, a quantity known to change over time. + /// This makes the project of computing a static value suitable for inclusion in a static, + /// generated file _excitingly unstable_. + pub fn existential_weight() -> VoteWeight { + let existential_deposit = >>::minimum_balance(); + let issuance = >>::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) + } +} From 202def0ba988b599bca37b1f2f4207cf56f147c6 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 15:35:53 +0200 Subject: [PATCH 52/82] write helper functions to emit voter bags module --- Cargo.lock | 20 ++++- frame/staking/Cargo.toml | 9 +- frame/staking/src/voter_bags/mod.rs | 126 +++++++++++++++++++++++++++- 3 files changed, 148 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c391dee935205..0576c4bfa07b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4621,6 +4621,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-format" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" +dependencies = [ + "arrayvec 0.4.12", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -5508,6 +5518,7 @@ dependencies = [ name = "pallet-staking" version = "3.0.0" dependencies = [ + "chrono", "frame-benchmarking", "frame-election-provider-support", "frame-support", @@ -5515,6 +5526,7 @@ dependencies = [ "git2", "hex", "log", + "num-format", "pallet-authorship", "pallet-balances", "pallet-session", @@ -10057,18 +10069,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ "proc-macro2", "quote", diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 1988e804c54e0..cd53cc5262afc 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -34,7 +34,9 @@ frame-benchmarking = { version = "3.1.0", default-features = false, path = "../b rand_chacha = { version = "0.2", default-features = false, optional = true } # Optional imports for making voter bags lists +chrono = { version = "0.4.19", optional = true } git2 = { version = "0.13.20", default-features = false, optional = true } +num-format = { version = "0.4.0", optional = true } [dev-dependencies] sp-storage = { version = "3.0.0", path = "../../primitives/storage" } @@ -74,7 +76,12 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] -make-bags = ["std", "git2"] +make-bags = [ + "chrono", + "git2", + "num-format", + "std", +] [[bin]] name = "make_bags" diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs index dd5869afe007a..c439f369744f3 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags/mod.rs @@ -609,8 +609,8 @@ pub enum VoterType { pub mod make_bags { use crate::{AccountIdOf, Config}; use frame_election_provider_support::VoteWeight; - use frame_support::traits::{Currency, CurrencyToVote}; - use std::path::{Path, PathBuf}; + use frame_support::traits::{Currency, CurrencyToVote, Get}; + use std::{io::Write, path::{Path, PathBuf}}; /// Return the path to a header file used in this repository if is exists. /// @@ -638,4 +638,126 @@ pub mod make_bags { let issuance = >>::total_issuance(); T::CurrencyToVote::to_vote(existential_deposit, issuance) } + + /// Compute the constant ratio for the thresholds. + /// + /// This ratio ensures that each bag, with the possible exceptions of certain small ones and the + /// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` + /// space. + pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { + (((VoteWeight::MAX - existential_weight) as f64).ln() / (n_bags as f64)).exp() + } + + /// Compute the list of bag thresholds. + /// + /// Returns a list of exactly `n_bags` elements, except in the case of overflow. + /// The first element is always `existential_weight`. + /// The last element is always `VoteWeight::MAX`. + /// + /// All other elements are computed from the previous according to the formula + /// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1); + pub fn thresholds( + existential_weight: VoteWeight, + constant_ratio: f64, + n_bags: usize, + ) -> Vec { + const WEIGHT_LIMIT: f64 = VoteWeight::MAX as f64; + + let mut thresholds = Vec::with_capacity(n_bags); + + if n_bags > 1 { + thresholds.push(existential_weight); + } + + while n_bags > 0 && thresholds.len() < n_bags - 1 { + let last = thresholds.last().copied().unwrap_or(existential_weight); + let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); + if successor < WEIGHT_LIMIT { + thresholds.push(successor as VoteWeight); + } + } + + thresholds.push(VoteWeight::MAX); + + debug_assert_eq!(thresholds.len(), n_bags); + debug_assert!(n_bags == 0 || thresholds[0] == existential_weight); + debug_assert!(n_bags == 0 || thresholds[thresholds.len() - 1] == VoteWeight::MAX); + + thresholds + } + + /// Write a thresholds module to the path specified. + /// + /// The `output` path should terminate with a Rust module name, i.e. `foo/bar/thresholds.rs`. + /// + /// This generated module contains, in order: + /// + /// - The contents of the header file in this repository's root, if found. + /// - Module documentation noting that this is autogenerated and when. + /// - Some associated constants. + /// - The constant array of thresholds. + pub fn generate_thresholds_module(n_bags: usize, output: &Path) -> Result<(), std::io::Error> { + // ensure the file is accessable + if let Some(parent) = output.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)?; + } + } + + // copy the header file + if let Some(header_path) = path_to_header_file() { + std::fs::copy(header_path, output)?; + } + + // open an append buffer + let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; + let mut buf = std::io::BufWriter::new(file); + + // module docs + let now = chrono::Utc::now(); + writeln!(buf)?; + writeln!(buf, "//! Autogenerated voter bag thresholds.")?; + writeln!(buf, "//!")?; + writeln!(buf, "//! Generated on {}", now.to_rfc3339())?; + writeln!( + buf, + "//! for the {} runtime.", + ::Version::get().spec_name, + )?; + + // existential weight + let existential_weight = existential_weight::(); + writeln!(buf)?; + writeln!(buf, "/// Existential weight for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", existential_weight)?; + + // constant ratio + let constant_ratio = constant_ratio(existential_weight, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Constant ratio between bags for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const CONSTANT_RATIO: f64 = {};", constant_ratio)?; + + // thresholds + let thresholds = thresholds(existential_weight, constant_ratio, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; + writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; + let mut num_buf = num_format::Buffer::new(); + let format = num_format::CustomFormat::builder() + .grouping(num_format::Grouping::Standard) + .separator("_") + .build() + .expect("format described here meets all constraints"); + for threshold in thresholds { + num_buf.write_formatted(&threshold, &format); + writeln!(buf, " {},", num_buf.as_str())?; + } + writeln!(buf, "];")?; + + Ok(()) + } } From 4da82040493f670464ef768f177036de585c517b Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 17:27:28 +0200 Subject: [PATCH 53/82] WIP: demo generating voter bags for a realistic runtime This isn't yet done, becuase it seems to take a Very Long Time to run, and it really shouldn't. Need to look into that. Still, it's a lot closer than it was this morning. --- Cargo.lock | 19 +- Cargo.toml | 1 + bin/node/runtime/src/lib.rs | 2 + bin/node/runtime/voter-bags/Cargo.toml | 16 ++ .../node/runtime/voter-bags/src/main.rs | 42 ++-- frame/staking/Cargo.toml | 4 - .../src/{voter_bags/mod.rs => voter_bags.rs} | 35 ++- frame/staking/src/voter_bags/thresholds.rs | 227 ------------------ 8 files changed, 91 insertions(+), 255 deletions(-) create mode 100644 bin/node/runtime/voter-bags/Cargo.toml rename frame/staking/src/bin/make_bags.rs => bin/node/runtime/voter-bags/src/main.rs (53%) rename frame/staking/src/{voter_bags/mod.rs => voter_bags.rs} (96%) delete mode 100644 frame/staking/src/voter_bags/thresholds.rs diff --git a/Cargo.lock b/Cargo.lock index 0576c4bfa07b8..7fc5552cdbe35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4461,6 +4461,16 @@ dependencies = [ "substrate-wasm-builder", ] +[[package]] +name = "node-runtime-voter-bags" +version = "3.0.0" +dependencies = [ + "node-runtime", + "pallet-staking", + "sp-io", + "structopt", +] + [[package]] name = "node-template" version = "3.0.0" @@ -6764,14 +6774,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -6786,9 +6795,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "region" diff --git a/Cargo.toml b/Cargo.toml index f7552f0bbbc48..fb25a9f120784 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "bin/node/rpc", "bin/node/rpc-client", "bin/node/runtime", + "bin/node/runtime/voter-bags", "bin/node/testing", "bin/utils/chain-spec-builder", "bin/utils/subkey", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index fd7fd4213366f..8dcf3fa210258 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -481,6 +481,7 @@ parameter_types! { pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; pub OffchainRepeat: BlockNumber = 5; + pub const VoterBagThresholds: &'static [u64] = &[]; } use frame_election_provider_support::onchain; @@ -510,6 +511,7 @@ impl pallet_staking::Config for Runtime { type GenesisElectionProvider = onchain::OnChainSequentialPhragmen>; type WeightInfo = pallet_staking::weights::SubstrateWeight; + type VoterBagThresholds = VoterBagThresholds; } parameter_types! { diff --git a/bin/node/runtime/voter-bags/Cargo.toml b/bin/node/runtime/voter-bags/Cargo.toml new file mode 100644 index 0000000000000..38f58ba9d4bd0 --- /dev/null +++ b/bin/node/runtime/voter-bags/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "node-runtime-voter-bags" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Voter Bag generation script for pallet-staking and node-runtime." +readme = "README.md" + +[dependencies] +node-runtime = { version = "2.0.0", path = ".." } +pallet-staking = { version = "3.0.0", path = "../../../../frame/staking", features = ["make-bags"] } +sp-io = { version = "3.0.0", path = "../../../../primitives/io" } +structopt = "0.3.21" diff --git a/frame/staking/src/bin/make_bags.rs b/bin/node/runtime/voter-bags/src/main.rs similarity index 53% rename from frame/staking/src/bin/make_bags.rs rename to bin/node/runtime/voter-bags/src/main.rs index a5773c59522c3..56e5f47cb1b00 100644 --- a/frame/staking/src/bin/make_bags.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,24 +17,32 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -const N_BAGS: usize = 200; - -fn main() { - let ratio = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); - println!("pub const CONSTANT_RATIO: f64 = {};", ratio); - - let mut thresholds = Vec::with_capacity(N_BAGS as usize); - - while thresholds.len() < N_BAGS as usize { - let prev_item: u64 = thresholds.last().copied().unwrap_or_default(); - let item = (prev_item as f64 * ratio).round().max(prev_item as f64 + 1.0); - thresholds.push(item as u64); +use pallet_staking::voter_bags::make_bags::generate_thresholds_module; +use std::path::PathBuf; +use structopt::{clap::arg_enum, StructOpt}; + +arg_enum!{ + #[derive(Debug)] + enum Runtime { + Node, } +} - *thresholds.last_mut().unwrap() = u64::MAX; - - println!("pub const THRESHOLDS: [u64; {}] = {:#?};", N_BAGS, thresholds); +#[derive(Debug, StructOpt)] +struct Opt { + /// How many bags to generate. + #[structopt( + long, + default_value = "200", + )] + n_bags: usize, + + /// Where to write the output. + output: PathBuf, +} - debug_assert_eq!(thresholds.len(), N_BAGS as usize); - debug_assert_eq!(*thresholds.last().unwrap(), u64::MAX); +fn main() -> Result<(), std::io::Error> { + let Opt {n_bags, output} = Opt::from_args(); + let mut ext = sp_io::TestExternalities::new_empty(); + ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index cd53cc5262afc..d69d1d4da82c3 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -82,7 +82,3 @@ make-bags = [ "num-format", "std", ] - -[[bin]] -name = "make_bags" -required-features = ["make-bags"] diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags.rs similarity index 96% rename from frame/staking/src/voter_bags/mod.rs rename to frame/staking/src/voter_bags.rs index c439f369744f3..b0d6ed9519475 100644 --- a/frame/staking/src/voter_bags/mod.rs +++ b/frame/staking/src/voter_bags.rs @@ -21,8 +21,6 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -pub mod thresholds; - use crate::{ AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, VotingDataOf, slashing::SlashingSpans, @@ -605,6 +603,39 @@ pub enum VoterType { } /// Support code to ease the process of generating voter bags. +/// +/// The process of adding voter bags to a runtime requires only four steps. +/// +/// 1. Update the runtime definition. +/// +/// ```ignore +/// parameter_types!{ +/// pub const VoterBagThresholds: &'static [u64] = &[]; +/// } +/// +/// impl pallet_staking::Config for Runtime { +/// // +/// type VoterBagThresholds = VoterBagThresholds; +/// } +/// ``` +/// +/// 2. Write a little program to generate the definitions. This can be a near-identical copy of +/// `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime +/// definitions with the +/// +/// 3. Run that program: +/// +/// ```sh,notrust +/// $ cargo run -p node-runtime-voter-bags -- bin/node/runtime/src/voter_bags.rs +/// ``` +/// +/// 4. Update the runtime definition. +/// +/// ```diff,notrust +/// + mod voter_bags; +/// - pub const VoterBagThresholds: &'static [u64] = &[]; +/// + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; +/// ``` #[cfg(feature = "make-bags")] pub mod make_bags { use crate::{AccountIdOf, Config}; diff --git a/frame/staking/src/voter_bags/thresholds.rs b/frame/staking/src/voter_bags/thresholds.rs deleted file mode 100644 index 37b889405ac19..0000000000000 --- a/frame/staking/src/voter_bags/thresholds.rs +++ /dev/null @@ -1,227 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Generated voter bag thresholds. - -/// Ratio between adjacent bags; -#[cfg(any(test, feature = "std"))] -#[allow(unused)] -pub const CONSTANT_RATIO: f64 = 1.2483305489016119; - -/// Upper thresholds for each bag. -pub const THRESHOLDS: [u64; 200] = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 11, - 13, - 16, - 19, - 23, - 28, - 34, - 42, - 52, - 64, - 79, - 98, - 122, - 152, - 189, - 235, - 293, - 365, - 455, - 567, - 707, - 882, - 1101, - 1374, - 1715, - 2140, - 2671, - 3334, - 4161, - 5194, - 6483, - 8092, - 10101, - 12609, - 15740, - 19648, - 24527, - 30617, - 38220, - 47711, - 59559, - 74349, - 92812, - 115860, - 144631, - 180547, - 225382, - 281351, - 351219, - 438437, - 547314, - 683228, - 852894, - 1064693, - 1329088, - 1659141, - 2071156, - 2585487, - 3227542, - 4029039, - 5029572, - 6278568, - 7837728, - 9784075, - 12213759, - 15246808, - 19033056, - 23759545, - 29659765, - 37025190, - 46219675, - 57697432, - 72025466, - 89911589, - 112239383, - 140111850, - 174905902, - 218340380, - 272560966, - 340246180, - 424739700, - 530215542, - 661884258, - 826250339, - 1031433539, - 1287569995, - 1607312958, - 2006457867, - 2504722650, - 3126721800, - 3903182340, - 4872461752, - 6082442853, - 7592899225, - 9478448057, - 11832236265, - 14770541991, - 18438518791, - 23017366283, - 28733281486, - 35868633049, - 44775910382, - 55895136784, - 69775606782, - 87103021514, - 108733362657, - 135735178289, - 169442369618, - 211520086272, - 264046985399, - 329617918218, - 411472116776, - 513653213392, - 641208997818, - 800440780206, - 999214678517, - 1247350208103, - 1557105369953, - 1943782201171, - 2426482702132, - 3029052483452, - 3781258749319, - 4720260810076, - 5892445768000, - 7355720059940, - 9182370059991, - 11462633057206, - 14309155016159, - 17862555335640, - 22298373506924, - 27835740839511, - 34748205641269, - 43377246621511, - 54149142084871, - 67596028261358, - 84382187063069, - 105336861893959, - 131495222627659, - 164149503440725, - 204912839732087, - 255798957699744, - 319321653273781, - 398618974707429, - 497608243499122, - 621179571745225, - 775437435763184, - 968002239825113, - 1208386767378873, - 1508466116607513, - 1883064335344139, - 2350686735357198, - 2934434062644189, - 3663143684136207, - 4572814165923224, - 5708383617772005, - 7125949654914296, - 8895540644164415, - 11104575135106362, - 13862180373726516, - 17304583234907174, - 21601839888145304, - 26966236644853160, - 33662776992680304, - 42022272880825152, - 52457686971413784, - 65484533171133904, - 81746343238087392, - 102046457525101200, - 127387710335774608, - 159021970366777056, - 198511983555374656, - 247808573395228608, - 309347012448991104, - 386167325851522816, - 482064469848099072, - 601775804251442048, - 751215120036911616, - 937764783138868096, - 1170640426476344320, - 1461346206149632000, - 1824243111658058240, - 2277258404906088192, - 2842771234587226112, - 3548718175673985024, - 4429973308136232448, - 5530071011365192704, - 6903356581082402816, - 8617670910126150656, - 10757701857491230720, - 13429167864681918464, - 18446744073709551615, -]; From 498ab6528cbacadaef1f7b40f7a8a571b422b26f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 2 Jul 2021 17:29:40 +0200 Subject: [PATCH 54/82] rm unnecessary arg_enum --- bin/node/runtime/voter-bags/src/main.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 56e5f47cb1b00..2d14539cf21fb 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -19,14 +19,7 @@ use pallet_staking::voter_bags::make_bags::generate_thresholds_module; use std::path::PathBuf; -use structopt::{clap::arg_enum, StructOpt}; - -arg_enum!{ - #[derive(Debug)] - enum Runtime { - Node, - } -} +use structopt::StructOpt; #[derive(Debug, StructOpt)] struct Opt { From 734ea86d569ca648afa65f46ad6bae95dfdcf5d0 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:19:37 +0200 Subject: [PATCH 55/82] fix voter bags math Turns out that when you're working in exponential space, you need to divide, not subtract, in order to keep the math working properly. Also neaten up the output a little bit to make it easier to read. --- frame/staking/src/voter_bags.rs | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index b0d6ed9519475..88973b62c114d 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -621,7 +621,7 @@ pub enum VoterType { /// /// 2. Write a little program to generate the definitions. This can be a near-identical copy of /// `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime -/// definitions with the +/// definitions with the various calculations here. /// /// 3. Run that program: /// @@ -659,6 +659,15 @@ pub mod make_bags { None } + /// Create an underscore formatter: a formatter which inserts `_` every 3 digits of a number. + fn underscore_formatter() -> num_format::CustomFormat { + num_format::CustomFormat::builder() + .grouping(num_format::Grouping::Standard) + .separator("_") + .build() + .expect("format described here meets all constraints") + } + /// Compute the existential weight for the specified configuration. /// /// Note that this value depends on the current issuance, a quantity known to change over time. @@ -676,7 +685,7 @@ pub mod make_bags { /// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` /// space. pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { - (((VoteWeight::MAX - existential_weight) as f64).ln() / (n_bags as f64)).exp() + ((VoteWeight::MAX as f64 / existential_weight as f64).ln() / ((n_bags - 1) as f64)).exp() } /// Compute the list of bag thresholds. @@ -705,6 +714,9 @@ pub mod make_bags { let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); if successor < WEIGHT_LIMIT { thresholds.push(successor as VoteWeight); + } else { + eprintln!("unexpectedly exceeded weight limit; breaking threshold generation loop"); + break } } @@ -744,6 +756,10 @@ pub mod make_bags { let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; let mut buf = std::io::BufWriter::new(file); + // create underscore formatter and format buffer + let mut num_buf = num_format::Buffer::new(); + let format = underscore_formatter(); + // module docs let now = chrono::Utc::now(); writeln!(buf)?; @@ -758,11 +774,12 @@ pub mod make_bags { // existential weight let existential_weight = existential_weight::(); + num_buf.write_formatted(&existential_weight, &format); writeln!(buf)?; writeln!(buf, "/// Existential weight for this runtime.")?; writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; writeln!(buf, "#[allow(unused)]")?; - writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", existential_weight)?; + writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", num_buf.as_str())?; // constant ratio let constant_ratio = constant_ratio(existential_weight, n_bags); @@ -777,15 +794,10 @@ pub mod make_bags { writeln!(buf)?; writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; - let mut num_buf = num_format::Buffer::new(); - let format = num_format::CustomFormat::builder() - .grouping(num_format::Grouping::Standard) - .separator("_") - .build() - .expect("format described here meets all constraints"); for threshold in thresholds { num_buf.write_formatted(&threshold, &format); - writeln!(buf, " {},", num_buf.as_str())?; + // u64::MAX, with spacers every 3 digits, is 26 characters wide + writeln!(buf, " {:>26},", num_buf.as_str())?; } writeln!(buf, "];")?; From 3cfa8cf8f7004f747e6f863b6dc5612bf99cb95c Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:31:21 +0200 Subject: [PATCH 56/82] add computed voter bags thresholds to node --- bin/node/runtime/src/lib.rs | 5 +- bin/node/runtime/src/voter_bags.rs | 235 +++++++++++++++++++++++++++++ 2 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 bin/node/runtime/src/voter_bags.rs diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 8dcf3fa210258..c4c394175d446 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -93,6 +93,9 @@ pub mod constants; use constants::{time::*, currency::*}; use sp_runtime::generic::Era; +/// Generated voter bag information. +mod voter_bags; + // Make the WASM binary available. #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); @@ -481,7 +484,7 @@ parameter_types! { pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; pub OffchainRepeat: BlockNumber = 5; - pub const VoterBagThresholds: &'static [u64] = &[]; + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; } use frame_election_provider_support::onchain; diff --git a/bin/node/runtime/src/voter_bags.rs b/bin/node/runtime/src/voter_bags.rs new file mode 100644 index 0000000000000..f8f6201e94dc3 --- /dev/null +++ b/bin/node/runtime/src/voter_bags.rs @@ -0,0 +1,235 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated voter bag thresholds. +//! +//! Generated on 2021-07-05T09:17:40.469754927+00:00 +//! for the node runtime. + +/// Existential weight for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const EXISTENTIAL_WEIGHT: u64 = 100_000_000_000_000; + +/// Constant ratio between bags for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 1.0628253590743408; + +/// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 200] = [ + 100_000_000_000_000, + 106_282_535_907_434, + 112_959_774_389_150, + 120_056_512_776_105, + 127_599_106_300_477, + 135_615_565_971_369, + 144_135_662_599_590, + 153_191_037_357_827, + 162_815_319_286_803, + 173_044_250_183_800, + 183_915_817_337_347, + 195_470_394_601_017, + 207_750_892_330_229, + 220_802_916_738_890, + 234_674_939_267_673, + 249_418_476_592_914, + 265_088_281_944_639, + 281_742_548_444_211, + 299_443_125_216_738, + 318_255_747_080_822, + 338_250_278_668_647, + 359_500_973_883_001, + 382_086_751_654_776, + 406_091_489_025_036, + 431_604_332_640_068, + 458_720_029_816_222, + 487_539_280_404_019, + 518_169_110_758_247, + 550_723_271_202_866, + 585_322_658_466_782, + 622_095_764_659_305, + 661_179_154_452_653, + 702_717_972_243_610, + 746_866_481_177_808, + 793_788_636_038_393, + 843_658_692_126_636, + 896_661_852_395_681, + 952_994_955_240_703, + 1_012_867_205_499_736, + 1_076_500_951_379_881, + 1_144_132_510_194_192, + 1_216_013_045_975_769, + 1_292_409_502_228_280, + 1_373_605_593_276_862, + 1_459_902_857_901_004, + 1_551_621_779_162_291, + 1_649_102_974_585_730, + 1_752_708_461_114_642, + 1_862_822_999_536_805, + 1_979_855_523_374_646, + 2_104_240_657_545_975, + 2_236_440_332_435_128, + 2_376_945_499_368_703, + 2_526_277_953_866_680, + 2_684_992_273_439_945, + 2_853_677_877_130_641, + 3_032_961_214_443_876, + 3_223_508_091_799_862, + 3_426_026_145_146_232, + 3_641_267_467_913_124, + 3_870_031_404_070_482, + 4_113_167_516_660_186, + 4_371_578_742_827_277, + 4_646_224_747_067_156, + 4_938_125_485_141_739, + 5_248_364_991_899_922, + 5_578_095_407_069_235, + 5_928_541_253_969_291, + 6_301_003_987_036_955, + 6_696_866_825_051_405, + 7_117_599_888_008_300, + 7_564_765_656_719_910, + 8_040_024_775_416_580, + 8_545_142_218_898_723, + 9_081_993_847_142_344, + 9_652_573_371_700_016, + 10_258_999_759_768_490, + 10_903_525_103_419_522, + 11_588_542_983_217_942, + 12_316_597_357_287_042, + 13_090_392_008_832_678, + 13_912_800_587_211_472, + 14_786_877_279_832_732, + 15_715_868_154_526_436, + 16_703_223_214_499_558, + 17_752_609_210_649_358, + 18_867_923_258_814_856, + 20_053_307_312_537_008, + 21_313_163_545_075_252, + 22_652_170_697_804_756, + 24_075_301_455_707_600, + 25_587_840_914_485_432, + 27_195_406_207_875_088, + 28_903_967_368_057_400, + 30_719_869_496_628_636, + 32_649_856_328_471_220, + 34_701_095_276_033_064, + 36_881_204_047_022_752, + 39_198_278_934_370_992, + 41_660_924_883_519_016, + 44_278_287_448_695_240, + 47_060_086_756_856_400, + 50_016_653_605_425_536, + 53_158_967_827_883_320, + 56_498_699_069_691_424, + 60_048_250_125_977_912, + 63_820_803_001_928_304, + 67_830_367_866_937_216, + 72_091_835_084_322_176, + 76_621_030_509_822_880, + 81_434_774_264_248_528, + 86_550_943_198_537_824, + 91_988_537_283_208_848, + 97_767_750_168_749_840, + 103_910_044_178_992_000, + 110_438_230_015_967_792, + 117_376_551_472_255_616, + 124_750_775_465_407_920, + 132_588_287_728_824_640, + 140_918_194_514_440_064, + 149_771_430_684_917_568, + 159_180_874_596_775_264, + 169_181_470_201_085_280, + 179_810_356_815_193_344, + 191_107_007_047_393_216, + 203_113_373_386_768_288, + 215_874_044_002_592_672, + 229_436_408_331_885_600, + 243_850_833_070_063_392, + 259_170_849_218_267_264, + 275_453_350_882_006_752, + 292_758_806_559_399_232, + 311_151_483_703_668_992, + 330_699_687_393_865_920, + 351_476_014_000_157_824, + 373_557_620_785_735_808, + 397_026_512_446_556_096, + 421_969_845_653_044_224, + 448_480_252_724_740_928, + 476_656_185_639_923_904, + 506_602_281_657_757_760, + 538_429_751_910_786_752, + 572_256_794_410_890_176, + 608_209_033_002_485_632, + 646_419_983_893_124_352, + 687_031_551_494_039_552, + 730_194_555_412_054_016, + 776_069_290_549_944_960, + 824_826_122_395_314_176, + 876_646_119_708_695_936, + 931_721_726_960_522_368, + 990_257_479_014_182_144, + 1_052_470_760_709_299_712, + 1_118_592_614_166_106_112, + 1_188_868_596_808_997_376, + 1_263_559_693_295_730_432, + 1_342_943_284_738_898_688, + 1_427_314_178_819_094_784, + 1_516_985_704_615_302_400, + 1_612_290_876_218_400_768, + 1_713_583_629_449_105_408, + 1_821_240_136_273_157_632, + 1_935_660_201_795_120_128, + 2_057_268_749_018_809_600, + 2_186_517_396_888_336_384, + 2_323_886_137_470_138_880, + 2_469_885_118_504_583_168, + 2_625_056_537_947_004_416, + 2_789_976_657_533_970_944, + 2_965_257_942_852_572_160, + 3_151_551_337_860_326_400, + 3_349_548_682_302_620_672, + 3_559_985_281_005_267_968, + 3_783_642_634_583_792_128, + 4_021_351_341_710_503_936, + 4_273_994_183_717_548_544, + 4_542_509_402_991_247_872, + 4_827_894_187_332_742_144, + 5_131_208_373_224_844_288, + 5_453_578_381_757_959_168, + 5_796_201_401_831_965_696, + 6_160_349_836_169_256_960, + 6_547_376_026_650_146_816, + 6_958_717_276_519_173_120, + 7_395_901_188_113_309_696, + 7_860_551_335_934_872_576, + 8_354_393_296_137_270_272, + 8_879_261_054_815_360_000, + 9_437_103_818_898_946_048, + 10_029_993_254_943_105_024, + 10_660_131_182_698_121_216, + 11_329_857_752_030_707_712, + 12_041_660_133_563_240_448, + 12_798_181_755_305_525_248, + 13_602_232_119_581_272_064, + 14_456_797_236_706_498_560, + 15_365_050_714_167_523_328, + 16_330_365_542_480_556_032, + 17_356_326_621_502_140_416, + 18_446_744_073_709_551_615, +]; From a05df6996cdbc8f072ba7690a268775c03f0f3de Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:47:28 +0200 Subject: [PATCH 57/82] fixup some docs --- frame/staking/src/lib.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 7f6b5402cff45..e3fc886c0c93f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -968,8 +968,9 @@ pub mod pallet { /// voter's weight is less than or equal to its upper threshold. /// /// When voters are iterated, higher bags are iterated completely before lower bags. This - /// that iteration is _semi-sorted_: voters of higher weight tend to come before voters of - /// lower weight, but peer voters within a particular bag are sorted in insertion order. + /// means that iteration is _semi-sorted_: voters of higher weight tend to come before + /// voters of lower weight, but peer voters within a particular bag are sorted in insertion + /// order. /// /// # Expressing the constant /// @@ -987,8 +988,8 @@ pub mod pallet { /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * /// constant_ratio).max(threshold[k] + 1)` for all `k`. /// - /// Given the desire to compute `N` bags, the constant ratio can be computed with - /// `exp(ln(VoterBags::MAX) / N)`. + /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use + /// them, the `make-bags` feature must be enabled. /// /// # Examples /// @@ -1004,8 +1005,9 @@ pub mod pallet { /// /// # Migration /// - /// In the event that this list ever changes, the `VoterList` data structure will need to be - /// regenerated. + /// In the event that this list ever changes, a copy of the old bags list must be retained. + /// With that `VoterList::migrate` can be called, which will perform the appropriate + /// migration. #[pallet::constant] type VoterBagThresholds: Get<&'static [VoteWeight]>; } From 2838b83ae63d94d57f2a20b6b54d00ba62589a13 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 11:48:06 +0200 Subject: [PATCH 58/82] iter from large bags to small, fulfuilling the contract --- frame/staking/src/voter_bags.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 88973b62c114d..559b84eb7adb4 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -108,6 +108,7 @@ impl VoterList { pub fn iter() -> impl Iterator> { T::VoterBagThresholds::get() .iter() + .rev() .copied() .filter_map(Bag::get) .flat_map(|bag| bag.iter()) From 3f177243bf35818586958cb74b26efda3177974d Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 14:14:59 +0200 Subject: [PATCH 59/82] make tests compile --- bin/node/runtime/voter-bags/src/main.rs | 35 ++++++++- frame/staking/Cargo.toml | 14 +++- frame/staking/src/benchmarking.rs | 2 +- frame/staking/src/lib.rs | 4 +- frame/staking/src/mock.rs | 9 ++- frame/staking/src/mock/voter_bags.rs | 100 ++++++++++++++++++++++++ frame/staking/src/tests.rs | 4 +- frame/staking/src/voter_bags.rs | 6 +- 8 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 frame/staking/src/mock/voter_bags.rs diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 2d14539cf21fb..fbd86adb185e0 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -18,8 +18,26 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. use pallet_staking::voter_bags::make_bags::generate_thresholds_module; -use std::path::PathBuf; -use structopt::StructOpt; +use pallet_staking::mock::Test; +use std::path::{Path, PathBuf}; +use structopt::{StructOpt, clap::arg_enum}; + +arg_enum!{ + #[derive(Debug)] + enum Runtime { + Node, + StakingMock, + } +} + +impl Runtime { + fn generate_thresholds(&self) -> Box Result<(), std::io::Error>> { + match self { + Runtime::Node => Box::new(generate_thresholds_module::), + Runtime::StakingMock => Box::new(generate_thresholds_module::), + } + } +} #[derive(Debug, StructOpt)] struct Opt { @@ -30,12 +48,21 @@ struct Opt { )] n_bags: usize, + /// Which runtime to generate. + #[structopt( + long, + case_insensitive = true, + default_value = "Node", + possible_values = &Runtime::variants(), + )] + runtime: Runtime, + /// Where to write the output. output: PathBuf, } fn main() -> Result<(), std::io::Error> { - let Opt {n_bags, output} = Opt::from_args(); + let Opt {n_bags, output, runtime } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) + ext.execute_with(|| runtime.generate_thresholds()(n_bags, &output)) } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index d69d1d4da82c3..d4d25c86c3940 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -16,14 +16,19 @@ targets = ["x86_64-unknown-linux-gnu"] static_assertions = "1.1.0" serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +sp-core = { version = "3.0.0", path = "../../primitives/core", optional = true } sp-std = { version = "3.0.0", default-features = false, path = "../../primitives/std" } sp-io ={ version = "3.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "3.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "3.0.0", default-features = false, path = "../../primitives/staking" } +sp-tracing = { version = "3.0.0", path = "../../primitives/tracing", optional = true } frame-support = { version = "3.0.0", default-features = false, path = "../support" } frame-system = { version = "3.0.0", default-features = false, path = "../system" } -pallet-session = { version = "3.0.0", default-features = false, features = ["historical"], path = "../session" } pallet-authorship = { version = "3.0.0", default-features = false, path = "../authorship" } +pallet-balances = { version = "3.0.0", path = "../balances", optional = true } +pallet-session = { version = "3.0.0", default-features = false, features = ["historical"], path = "../session" } +pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve", optional = true } +pallet-timestamp = { version = "3.0.0", path = "../timestamp", optional = true } sp-application-crypto = { version = "3.0.0", default-features = false, path = "../../primitives/application-crypto" } frame-election-provider-support = { version = "3.0.0", default-features = false, path = "../election-provider-support" } log = { version = "0.4.14", default-features = false } @@ -45,7 +50,7 @@ sp-core = { version = "3.0.0", path = "../../primitives/core" } sp-npos-elections = { version = "3.0.0", path = "../../primitives/npos-elections", features = ["mocks"] } pallet-balances = { version = "3.0.0", path = "../balances" } pallet-timestamp = { version = "3.0.0", path = "../timestamp" } -pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } +pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } substrate-test-utils = { version = "3.0.0", path = "../../test-utils" } frame-benchmarking = { version = "3.1.0", path = "../benchmarking" } frame-election-provider-support = { version = "3.0.0", features = ["runtime-benchmarks"], path = "../election-provider-support" } @@ -80,5 +85,10 @@ make-bags = [ "chrono", "git2", "num-format", + "pallet-staking-reward-curve", + "pallet-balances", + "pallet-timestamp", + "sp-core", + "sp-tracing", "std", ] diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 208769ce1ccac..fbae528a95058 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -700,7 +700,7 @@ benchmarks! { ); ensure!( { - let origin_bag = Bag::::get(node.bag_idx).ok_or("origin bag not found")?; + let origin_bag = Bag::::get(node.bag_upper).ok_or("origin bag not found")?; origin_bag.iter().count() == 1 }, "stash should be the only node in origin bag", diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index e3fc886c0c93f..0c761e2528bd6 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -274,8 +274,8 @@ #![recursion_limit = "128"] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(test)] -mod mock; +#[cfg(any(test, feature = "make-bags"))] +pub mod mock; #[cfg(test)] mod tests; #[cfg(any(feature = "runtime-benchmarks", test))] diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index d9cc77653e986..a584b83c539c8 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,6 +17,12 @@ //! Test utilities +// This module needs to exist when the `make-bags` feature is enabled so that we can generate the +// appropriate thresholds, but we don't care if it's mostly unused in that case. +#![cfg_attr(feature = "make-bags", allow(unused))] + +mod voter_bags; + use crate::*; use crate as staking; use frame_support::{ @@ -244,7 +250,7 @@ impl onchain::Config for Test { } parameter_types! { - pub const VoterBagThresholds: &'static [VoteWeight] = &crate::voter_bags::thresholds::THRESHOLDS; + pub const VoterBagThresholds: &'static [VoteWeight] = &voter_bags::THRESHOLDS; } impl Config for Test { @@ -268,7 +274,6 @@ impl Config for Test { type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); type VoterBagThresholds = VoterBagThresholds; - type BagIdx = u8; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/mock/voter_bags.rs b/frame/staking/src/mock/voter_bags.rs new file mode 100644 index 0000000000000..453b03e36ab22 --- /dev/null +++ b/frame/staking/src/mock/voter_bags.rs @@ -0,0 +1,100 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated voter bag thresholds. +//! +//! Generated on 2021-07-05T12:08:52.871368217+00:00 +//! for the test runtime. + +/// Existential weight for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const EXISTENTIAL_WEIGHT: u64 = 1; + +/// Constant ratio between bags for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 2.0000000000000000; + +/// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 65] = [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1_024, + 2_048, + 4_096, + 8_192, + 16_384, + 32_768, + 65_536, + 131_072, + 262_144, + 524_288, + 1_048_576, + 2_097_152, + 4_194_304, + 8_388_608, + 16_777_216, + 33_554_432, + 67_108_864, + 134_217_728, + 268_435_456, + 536_870_912, + 1_073_741_824, + 2_147_483_648, + 4_294_967_296, + 8_589_934_592, + 17_179_869_184, + 34_359_738_368, + 68_719_476_736, + 137_438_953_472, + 274_877_906_944, + 549_755_813_888, + 1_099_511_627_776, + 2_199_023_255_552, + 4_398_046_511_104, + 8_796_093_022_208, + 17_592_186_044_416, + 35_184_372_088_832, + 70_368_744_177_664, + 140_737_488_355_328, + 281_474_976_710_656, + 562_949_953_421_312, + 1_125_899_906_842_624, + 2_251_799_813_685_248, + 4_503_599_627_370_496, + 9_007_199_254_740_992, + 18_014_398_509_481_984, + 36_028_797_018_963_968, + 72_057_594_037_927_936, + 144_115_188_075_855_872, + 288_230_376_151_711_744, + 576_460_752_303_423_488, + 1_152_921_504_606_846_976, + 2_305_843_009_213_693_952, + 4_611_686_018_427_387_904, + 9_223_372_036_854_775_808, + 18_446_744_073_709_551_615, +]; diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index e0e6b83d4b742..864fcb0c35683 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3909,7 +3909,7 @@ fn test_rebag() { let node = Node::::from_id(&stash).unwrap(); assert_eq!( { - let origin_bag = Bag::::get(node.bag_idx).unwrap(); + let origin_bag = Bag::::get(node.bag_upper).unwrap(); origin_bag.iter().count() }, 1, @@ -3919,7 +3919,7 @@ fn test_rebag() { assert!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); assert_ne!( { - let destination_bag = Bag::::get(other_node.bag_idx); + let destination_bag = Bag::::get(other_node.bag_upper); destination_bag.iter().count() }, 0, diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 559b84eb7adb4..1d748142f6ce7 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -556,14 +556,14 @@ impl Node { existed } - /// Get the index of the bag that this node _should_ be in, given its vote weight. + /// Get the upper threshold of the bag that this node _should_ be in, given its vote weight. /// /// This is a helper intended only for benchmarking and should not be used in production. #[cfg(any(test, feature = "runtime-benchmarks"))] pub fn proper_bag_for(&self) -> VoteWeight { let weight_of = crate::Pallet::::weight_of_fn(); let current_weight = weight_of(&self.voter.id); - notional_bag_for(current_weight) + notional_bag_for::(current_weight) } } @@ -788,7 +788,7 @@ pub mod make_bags { writeln!(buf, "/// Constant ratio between bags for this runtime.")?; writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; writeln!(buf, "#[allow(unused)]")?; - writeln!(buf, "pub const CONSTANT_RATIO: f64 = {};", constant_ratio)?; + writeln!(buf, "pub const CONSTANT_RATIO: f64 = {:.16};", constant_ratio)?; // thresholds let thresholds = thresholds(existential_weight, constant_ratio, n_bags); From b73fd7f2c46d4699739467b8c7b5d843e50b3f6e Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Mon, 5 Jul 2021 14:56:12 +0200 Subject: [PATCH 60/82] add VoterBagThresholds to some configs --- frame/babe/src/mock.rs | 1 + frame/grandpa/src/mock.rs | 1 + frame/offences/benchmarking/src/mock.rs | 1 + frame/session/benchmarking/src/mock.rs | 1 + 4 files changed, 4 insertions(+) diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 6c1cc89cf1ed0..3a18c11616c05 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -215,6 +215,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_offences::Config for Test { diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index ebe5996c9dab5..27f7b3d488437 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -221,6 +221,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_offences::Config for Test { diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index cd72780ec5ad2..84527ea4ab033 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -180,6 +180,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_im_online::Config for Test { diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 591e54f067bb5..6415dd8a9fede 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -185,6 +185,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl crate::Config for Test {} From 4861789b0bc0ca027879beb75b558ec0d8f6d203 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 6 Jul 2021 14:08:11 +0200 Subject: [PATCH 61/82] ensure that iteration covers all voters even with implied final bag --- frame/staking/src/voter_bags.rs | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 1d748142f6ce7..b10fdac4eb374 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -26,9 +26,9 @@ use crate::{ VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; -use frame_support::{DefaultNoBound, traits::Get}; +use frame_support::{DebugNoBound, DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; -use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, marker::PhantomData}; +use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, iter, marker::PhantomData}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; @@ -106,10 +106,22 @@ impl VoterList { /// /// Full iteration can be expensive; it's recommended to limit the number of items with `.take(n)`. pub fn iter() -> impl Iterator> { - T::VoterBagThresholds::get() - .iter() - .rev() - .copied() + // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to + // omit the final bound, we need to ensure that we explicitly include that threshold in the + // list. + // + // It's important to retain the ability to omit the final bound because it makes tests much + // easier; they can just configure `type VoterBagThresholds = ()`. + let thresholds = T::VoterBagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == Some(&VoteWeight::MAX) { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter.rev()) + } else { + // otherwise, insert it here. + Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) + }; + iter .filter_map(Bag::get) .flat_map(|bag| bag.iter()) } @@ -331,6 +343,7 @@ impl VoterList { /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. #[derive(DefaultNoBound, Encode, Decode)] +#[cfg_attr(feature = "std", derive(DebugNoBound))] pub struct Bag { head: Option>, tail: Option>, @@ -444,7 +457,7 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] +#[cfg_attr(feature = "std", derive(DebugNoBound))] pub struct Node { voter: Voter>, prev: Option>, From 59041b7250a4ebd95fdf76458d84de87b5b9511f Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 6 Jul 2021 14:30:22 +0200 Subject: [PATCH 62/82] use sp_std::boxed::Box; --- frame/staking/src/voter_bags.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index b10fdac4eb374..cc7e124fc7cbb 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -28,7 +28,12 @@ use crate::{ use codec::{Encode, Decode}; use frame_support::{DebugNoBound, DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; -use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, iter, marker::PhantomData}; +use sp_std::{ + boxed::Box, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + iter, + marker::PhantomData, +}; /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; From f5c8a4d7e5a7c05e38823d22750d1afa1f0db681 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Tue, 6 Jul 2021 15:29:02 +0200 Subject: [PATCH 63/82] fix unused import --- frame/staking/src/voter_bags.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index cc7e124fc7cbb..6e65120e7ba4e 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -26,7 +26,7 @@ use crate::{ VotingDataOf, slashing::SlashingSpans, }; use codec::{Encode, Decode}; -use frame_support::{DebugNoBound, DefaultNoBound, traits::Get}; +use frame_support::{DefaultNoBound, traits::Get}; use sp_runtime::SaturatedConversion; use sp_std::{ boxed::Box, @@ -348,7 +348,7 @@ impl VoterList { /// iteration so that there's no incentive to churn voter positioning to improve the chances of /// appearing within the voter set. #[derive(DefaultNoBound, Encode, Decode)] -#[cfg_attr(feature = "std", derive(DebugNoBound))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] pub struct Bag { head: Option>, tail: Option>, @@ -462,7 +462,7 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(DebugNoBound))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] pub struct Node { voter: Voter>, prev: Option>, From 5e05e3fd87ffa80d36a60cf07fc11a6e04a817fe Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 7 Jul 2021 10:47:08 +0200 Subject: [PATCH 64/82] add some more voter bags tests --- Cargo.lock | 35 +++++++++---------- frame/staking/Cargo.toml | 1 + frame/staking/src/voter_bags.rs | 60 ++++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec41ab10c5148..21ed44dce2826 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1701,7 +1701,7 @@ dependencies = [ "num-traits", "parity-scale-codec", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -1711,7 +1711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" dependencies = [ "byteorder", - "rand 0.8.3", + "rand 0.8.4", "rustc-hex", "static_assertions", ] @@ -3382,7 +3382,7 @@ dependencies = [ "libp2p-core", "libp2p-swarm", "log", - "rand 0.8.3", + "rand 0.8.4", "smallvec 1.6.1", "socket2 0.4.0", "void", @@ -4055,7 +4055,7 @@ dependencies = [ "num-complex", "num-rational 0.4.0", "num-traits", - "rand 0.8.3", + "rand 0.8.4", "rand_distr", "simba", "typenum", @@ -4919,7 +4919,7 @@ dependencies = [ "paste 1.0.4", "pretty_assertions 0.7.2", "pwasm-utils", - "rand 0.8.3", + "rand 0.8.4", "rand_pcg 0.3.0", "serde", "smallvec 1.6.1", @@ -5532,6 +5532,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.11.1", "paste 1.0.4", + "rand 0.8.4", "rand_chacha 0.2.2", "serde", "sp-application-crypto", @@ -5773,7 +5774,7 @@ dependencies = [ "log", "memmap2", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -6476,7 +6477,7 @@ checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ "env_logger 0.8.3", "log", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -6544,9 +6545,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha 0.3.0", @@ -6614,7 +6615,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051b398806e42b9cd04ad9ec8f81e355d0a382c543ac6672c62f5a5b452ef142" dependencies = [ "num-traits", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -7685,7 +7686,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.11.1", "prost", - "rand 0.8.3", + "rand 0.8.4", "sc-block-builder", "sc-client-api", "sc-finality-grandpa", @@ -8704,7 +8705,7 @@ dependencies = [ "futures 0.3.15", "httparse", "log", - "rand 0.8.3", + "rand 0.8.4", "sha-1 0.9.4", ] @@ -9552,7 +9553,7 @@ dependencies = [ "lazy_static", "nalgebra", "num-traits", - "rand 0.8.3", + "rand 0.8.4", ] [[package]] @@ -9956,7 +9957,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.3", + "rand 0.8.4", "redox_syscall 0.2.5", "remove_dir_all", "winapi 0.3.9", @@ -10626,7 +10627,7 @@ dependencies = [ "ipnet", "lazy_static", "log", - "rand 0.8.3", + "rand 0.8.4", "smallvec 1.6.1", "thiserror", "tinyvec", @@ -11292,7 +11293,7 @@ dependencies = [ "mach", "memoffset 0.6.1", "more-asserts", - "rand 0.8.3", + "rand 0.8.4", "region", "thiserror", "wasmtime-environ", @@ -11461,7 +11462,7 @@ dependencies = [ "log", "nohash-hasher", "parking_lot 0.11.1", - "rand 0.8.3", + "rand 0.8.4", "static_assertions", ] diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index d4d25c86c3940..8897262431606 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -54,6 +54,7 @@ pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-cur substrate-test-utils = { version = "3.0.0", path = "../../test-utils" } frame-benchmarking = { version = "3.1.0", path = "../benchmarking" } frame-election-provider-support = { version = "3.0.0", features = ["runtime-benchmarks"], path = "../election-provider-support" } +rand = "0.8.4" rand_chacha = { version = "0.2" } parking_lot = "0.11.1" hex = "0.4" diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 6e65120e7ba4e..1772414621361 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -758,7 +758,10 @@ pub mod make_bags { /// - Module documentation noting that this is autogenerated and when. /// - Some associated constants. /// - The constant array of thresholds. - pub fn generate_thresholds_module(n_bags: usize, output: &Path) -> Result<(), std::io::Error> { + pub fn generate_thresholds_module( + n_bags: usize, + output: &Path, + ) -> Result<(), std::io::Error> { // ensure the file is accessable if let Some(parent) = output.parent() { if !parent.exists() { @@ -823,3 +826,58 @@ pub mod make_bags { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::mock::{ExtBuilder, Staking, Test}; + use frame_support::traits::Currency; + use substrate_test_utils::assert_eq_uvec; + use super::*; + + const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; + + #[test] + fn voter_list_includes_genesis_accounts() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); + assert_eq_uvec!(GENESIS_VOTER_IDS, have_voters); + }); + } + + /// This tests the property that when iterating through the `VoterList`, we iterate from higher + /// bags to lower. + #[test] + fn iteration_is_semi_sorted() { + use rand::seq::SliceRandom; + let mut rng = rand::thread_rng(); + + // Randomly sort the list of voters. Later we'll give each of these a stake such that it + // fits into a different bag. + let voters = { + let mut v = vec![0; GENESIS_VOTER_IDS.len()]; + v.copy_from_slice(&GENESIS_VOTER_IDS); + v.shuffle(&mut rng); + v + }; + + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // initialize the voters' deposits + let existential_deposit = ::Currency::minimum_balance(); + let mut balance = existential_deposit + 1; + for voter_id in voters.iter().rev() { + ::Currency::make_free_balance_be(voter_id, balance); + let controller = Staking::bonded(voter_id).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = balance; + ledger.active = balance; + Staking::update_ledger(&controller, &ledger); + Staking::do_rebag(voter_id); + + balance *= 2; + } + + let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); + assert_eq!(voters, have_voters); + }); + } +} From dbf3a84c84da3ad8ca64e68aea720e9733252882 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Wed, 7 Jul 2021 13:44:15 +0200 Subject: [PATCH 65/82] file_header.txt --- frame/staking/src/voter_bags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 1772414621361..dcdba815cdaa6 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -669,7 +669,7 @@ pub mod make_bags { fn path_to_header_file() -> Option { let repo = git2::Repository::open_from_env().ok()?; let workdir = repo.workdir()?; - for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER"] { + for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { let path = workdir.join(file_name); if path.exists() { return Some(path); From d5e0c8dedebcea88b9dc792ea3075f8d0ae2ffb1 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 9 Jul 2021 10:08:04 +0200 Subject: [PATCH 66/82] integrity test to ensure min bag exceeds existential weight --- frame/staking/src/lib.rs | 11 +++++++++++ frame/staking/src/voter_bags.rs | 27 +++++++++++++++------------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index c87c0b53556ce..5b834283acc2e 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1568,6 +1568,17 @@ pub mod pallet { T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), "Voter bag thresholds must strictly increase", ); + + assert!( + { + let existential_weight = voter_bags::existential_weight::(); + T::VoterBagThresholds::get() + .first() + .map(|&lowest_threshold| lowest_threshold >= existential_weight) + .unwrap_or(true) + }, + "Smallest bag should not be smaller than existential weight", + ); }); } } diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index dcdba815cdaa6..ff6113a18c41a 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -621,6 +621,20 @@ pub enum VoterType { Nominator, } +/// Compute the existential weight for the specified configuration. +/// +/// Note that this value depends on the current issuance, a quantity known to change over time. +/// This makes the project of computing a static value suitable for inclusion in a static, +/// generated file _excitingly unstable_. +#[cfg(any(feature = "std", feature = "make-bags"))] +pub fn existential_weight() -> VoteWeight { + use frame_support::traits::{Currency, CurrencyToVote}; + + let existential_deposit = >>::minimum_balance(); + let issuance = >>::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) +} + /// Support code to ease the process of generating voter bags. /// /// The process of adding voter bags to a runtime requires only four steps. @@ -657,7 +671,7 @@ pub enum VoterType { /// ``` #[cfg(feature = "make-bags")] pub mod make_bags { - use crate::{AccountIdOf, Config}; + use crate::{AccountIdOf, Config, voter_bags::existential_weight}; use frame_election_provider_support::VoteWeight; use frame_support::traits::{Currency, CurrencyToVote, Get}; use std::{io::Write, path::{Path, PathBuf}}; @@ -687,17 +701,6 @@ pub mod make_bags { .expect("format described here meets all constraints") } - /// Compute the existential weight for the specified configuration. - /// - /// Note that this value depends on the current issuance, a quantity known to change over time. - /// This makes the project of computing a static value suitable for inclusion in a static, - /// generated file _excitingly unstable_. - pub fn existential_weight() -> VoteWeight { - let existential_deposit = >>::minimum_balance(); - let issuance = >>::total_issuance(); - T::CurrencyToVote::to_vote(existential_deposit, issuance) - } - /// Compute the constant ratio for the thresholds. /// /// This ratio ensures that each bag, with the possible exceptions of certain small ones and the From dbfb9512120a4cec144264729649464281d52429 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 9 Jul 2021 10:22:28 +0200 Subject: [PATCH 67/82] add more debug assertions about node list length --- frame/staking/src/voter_bags.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index ff6113a18c41a..04362be720d97 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -104,7 +104,18 @@ impl VoterList { /// Decode the length of the voter list. pub fn decode_len() -> Option { - crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) + let maybe_len = crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()); + debug_assert_eq!( + maybe_len.unwrap_or_default(), + crate::VoterNodes::::iter().count(), + "stored length must match count of nodes", + ); + debug_assert_eq!( + maybe_len.unwrap_or_default() as u32, + crate::CounterForNominators::::get() + crate::CounterForValidators::::get(), + "voter count must be sum of validator and nominator count", + ); + maybe_len } /// Iterate over all nodes in all bags in the voter list. From 06e69a8e1bbd8ca2cc9a82885a0bd811a0b6ee80 Mon Sep 17 00:00:00 2001 From: Peter Goodspeed-Niklaus Date: Fri, 9 Jul 2021 10:43:55 +0200 Subject: [PATCH 68/82] rm unused imports --- frame/staking/src/voter_bags.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 04362be720d97..6457770e632ea 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -682,9 +682,9 @@ pub fn existential_weight() -> VoteWeight { /// ``` #[cfg(feature = "make-bags")] pub mod make_bags { - use crate::{AccountIdOf, Config, voter_bags::existential_weight}; + use crate::{Config, voter_bags::existential_weight}; use frame_election_provider_support::VoteWeight; - use frame_support::traits::{Currency, CurrencyToVote, Get}; + use frame_support::traits::Get; use std::{io::Write, path::{Path, PathBuf}}; /// Return the path to a header file used in this repository if is exists. From 6f15cc3c0c11f67268b2dcfd38f487ff569cd102 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Sat, 10 Jul 2021 21:48:53 +0200 Subject: [PATCH 69/82] Kian enters --- frame/staking/src/lib.rs | 23 ++++++++++++++++++----- frame/staking/src/mock/voter_bags.rs | 1 + frame/staking/src/voter_bags.rs | 20 ++++++++++++++++---- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 5b834283acc2e..00458ea9f49b0 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -1907,7 +1907,6 @@ pub mod pallet { let controller = ensure_signed(origin)?; let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; Self::chill_stash(&ledger.stash); - VoterList::::remove(&ledger.stash); Ok(()) } @@ -2964,8 +2963,6 @@ impl Pallet { Self::do_remove_validator(stash); Self::do_remove_nominator(stash); - VoterList::::remove(stash); - frame_system::Pallet::::dec_consumers(stash); Ok(()) @@ -3089,6 +3086,10 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// If the nominator already exists, their nominations will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a nominator to the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { if !Nominators::::contains_key(who) { CounterForNominators::::mutate(|x| x.saturating_inc()) @@ -3102,6 +3103,10 @@ impl Pallet { /// and keep track of the `CounterForNominators`. /// /// Returns true if `who` was removed from `Nominators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_remove_nominator(who: &T::AccountId) -> bool { if Nominators::::contains_key(who) { Nominators::::remove(who); @@ -3114,10 +3119,14 @@ impl Pallet { } } - /// This function will add a validator to the `Validators` storage map, - /// and keep track of the `CounterForValidators`. + /// This function will add a validator to the `Validators` storage map, and keep track of the + /// `CounterForValidators`. /// /// If the validator already exists, their preferences will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { if !Validators::::contains_key(who) { CounterForValidators::::mutate(|x| x.saturating_inc()) @@ -3131,6 +3140,10 @@ impl Pallet { /// and keep track of the `CounterForValidators`. /// /// Returns true if `who` was removed from `Validators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. pub fn do_remove_validator(who: &T::AccountId) -> bool { if Validators::::contains_key(who) { Validators::::remove(who); diff --git a/frame/staking/src/mock/voter_bags.rs b/frame/staking/src/mock/voter_bags.rs index 453b03e36ab22..6177c615bc59a 100644 --- a/frame/staking/src/mock/voter_bags.rs +++ b/frame/staking/src/mock/voter_bags.rs @@ -31,6 +31,7 @@ pub const EXISTENTIAL_WEIGHT: u64 = 1; pub const CONSTANT_RATIO: f64 = 2.0000000000000000; /// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 65] = [5, 15, 25, 50, 100, 1000, 2000, 3000] pub const THRESHOLDS: [u64; 65] = [ 1, 2, diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 6457770e632ea..34efce5c59f02 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -120,7 +120,8 @@ impl VoterList { /// Iterate over all nodes in all bags in the voter list. /// - /// Full iteration can be expensive; it's recommended to limit the number of items with `.take(n)`. + /// Full iteration can be expensive; it's recommended to limit the number of items with + /// `.take(n)`. pub fn iter() -> impl Iterator> { // We need a touch of special handling here: because we permit `T::VoterBagThresholds` to // omit the final bound, we need to ensure that we explicitly include that threshold in the @@ -157,14 +158,14 @@ impl VoterList { } /// Insert a new voter into the appropriate bag in the voter list. - pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { + fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { Self::insert_many(sp_std::iter::once(voter), weight_of); } /// Insert several voters into the appropriate bags in the voter list. /// /// This is more efficient than repeated calls to `Self::insert`. - pub fn insert_many( + fn insert_many( voters: impl IntoIterator>, weight_of: impl Fn(&T::AccountId) -> VoteWeight, ) -> u32 { @@ -459,7 +460,17 @@ impl Bag { /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call /// `self.put()` and `node.put()` after use. fn remove_node(&mut self, node: &Node) { - node.excise(); + // TODO: we could merge this function here. + // node.excise(); + if let Some(mut prev) = node.prev() { + prev.next = self.next.clone(); + prev.put(); + } + if let Some(mut next) = node.next() { + next.prev = self.prev.clone(); + next.put(); + } + // IDEA: debug_assert! prev.next.prev == self // clear the bag head/tail pointers as necessary if self.head.as_ref() == Some(&node.voter.id) { @@ -487,6 +498,7 @@ pub struct Node { impl Node { /// Get a node by bag idx and account id. pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { + // debug_assert!(bag_upper is in Threshold) crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { node.bag_upper = bag_upper; node From 1b545e482e1fd3912cf09a7ff4fc753003736eee Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Mon, 12 Jul 2021 10:02:22 +0200 Subject: [PATCH 70/82] Update frame/election-provider-support/src/onchain.rs Co-authored-by: Zeke Mostov <32168567+emostov@users.noreply.github.com> --- frame/election-provider-support/src/onchain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index 89358fce12af8..47add4e70461c 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -78,7 +78,7 @@ impl ElectionProvider for OnChainSequen Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; let stake_map: BTreeMap = - voters.iter().map(|(v, s, _)| (v.clone(), *s)).collect(); +voters.iter().map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)).collect(); let stake_of = |w: &T::AccountId| -> VoteWeight { stake_map.get(w).cloned().unwrap_or_default() From e0a00fd187343fd2c28d2060268b82dd1f9b4cce Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 21 Jul 2021 12:23:06 -0700 Subject: [PATCH 71/82] Suggestions for #9081 (Store voters in unsorted bags) (#9328) * Add some debug asserts to node::get and remove_node * Improve the debug asserts in remove_node * improve debug asserts * Space * Remove bad assertions * Tests: WIP take_works * Take test * Doc comment * Apply suggestions from code review Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> * Test storage is cleaned up; * formatting * Switch to simpler thresholds * Update the storage cleanup test * Remove hardcoded values from benchmark to make it more robust * Fix tests to acces bags properly * Sanity check WIP; tests failing * Update sanity checks to be more correct * Improve storage cleanup tests * WIP remote_ext_tests * Some notes on next steps * Remove some stuff that was for remote-ext tests * Some more cleanup to reduce diff * More :clean: * Mo cleanin * small fix * A lot of changes from kian Co-authored-by: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Co-authored-by: kianenigma --- bin/node/runtime/voter-bags/src/main.rs | 2 +- frame/staking/src/benchmarking.rs | 10 +- frame/staking/src/lib.rs | 57 ++- frame/staking/src/mock.rs | 40 +- frame/staking/src/mock/voter_bags.rs | 101 ----- frame/staking/src/tests.rs | 34 +- frame/staking/src/voter_bags.rs | 471 +++++++++++++++++++++--- 7 files changed, 519 insertions(+), 196 deletions(-) delete mode 100644 frame/staking/src/mock/voter_bags.rs diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index fbd86adb185e0..9ae50914d6927 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -62,7 +62,7 @@ struct Opt { } fn main() -> Result<(), std::io::Error> { - let Opt {n_bags, output, runtime } = Opt::from_args(); + let Opt { n_bags, output, runtime } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); ext.execute_with(|| runtime.generate_thresholds()(n_bags, &output)) } diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index fbae528a95058..3eeb739371517 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -671,11 +671,15 @@ benchmarks! { // Clean up any existing state. clear_validators_and_nominators::(); + let thresholds = T::VoterBagThresholds::get(); + // stash controls the node account - let (stash, controller) = make_validator(USER_SEED, 100)?; + let bag0_thresh = thresholds[0]; + let (stash, controller) = make_validator(USER_SEED, bag0_thresh as u32)?; - // create another validator with 3x the stake - let (other_stash, _) = make_validator(USER_SEED + 1, 300)?; + // create another validator with more stake + let bag2_thresh = thresholds[2]; + let (other_stash, _) = make_validator(USER_SEED + 1, bag2_thresh as u32)?; // update the stash account's value/weight // diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 00458ea9f49b0..f3eb9d68968f2 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -780,6 +780,7 @@ pub mod migrations { log!(info, "Migrating staking to Releases::V8_0_0"); let migrated = VoterList::::regenerate(); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); StorageVersion::::put(Releases::V8_0_0); log!( @@ -1313,7 +1314,7 @@ pub mod pallet { /// How many voters are registered. #[pallet::storage] - pub(crate) type VoterCount = StorageValue<_, u32, ValueQuery>; + pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; /// Which bag currently contains a particular voter. /// @@ -1397,32 +1398,62 @@ pub mod pallet { MinNominatorBond::::put(self.min_nominator_bond); MinValidatorBond::::put(self.min_validator_bond); + let mut num_voters: u32 = 0; for &(ref stash, ref controller, balance, ref status) in &self.stakers { + log!( + trace, + "inserting genesis staker: {:?} => {:?} => {:?}", + stash, + balance, + status + ); assert!( T::Currency::free_balance(&stash) >= balance, "Stash does not have enough balance to bond." ); - let _ = >::bond( + + if let Err(why) = >::bond( T::Origin::from(Some(stash.clone()).into()), T::Lookup::unlookup(controller.clone()), balance, RewardDestination::Staked, - ); - let _ = match status { + ) { + // TODO: later on, fix all the tests that trigger these warnings, and + // make these assertions. Genesis stakers should all be correct! + log!(warn, "failed to bond staker at genesis: {:?}.", why); + continue; + } + match status { StakerStatus::Validator => { - >::validate( + if let Err(why) = >::validate( T::Origin::from(Some(controller.clone()).into()), Default::default(), - ) + ) { + log!(warn, "failed to validate staker at genesis: {:?}.", why); + } else { + num_voters +=1 ; + } }, StakerStatus::Nominator(votes) => { - >::nominate( + if let Err(why) = >::nominate( T::Origin::from(Some(controller.clone()).into()), votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), - ) - }, _ => Ok(()) + ) { + log!(warn, "failed to nominate staker at genesis: {:?}.", why); + } else { + num_voters += 1; + } + } + _ => () }; } + + // all voters are inserted sanely. + assert_eq!( + CounterForVoters::::get(), + num_voters, + "not all genesis stakers were inserted into bags, something is wrong." + ); } } @@ -3096,7 +3127,7 @@ impl Pallet { } Nominators::::insert(who, nominations); VoterList::::insert_as(who, VoterType::Nominator); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); } /// This function will remove a nominator from the `Nominators` storage map, @@ -3112,7 +3143,7 @@ impl Pallet { Nominators::::remove(who); CounterForNominators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); true } else { false @@ -3133,7 +3164,7 @@ impl Pallet { } Validators::::insert(who, prefs); VoterList::::insert_as(who, VoterType::Validator); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); } /// This function will remove a validator from the `Validators` storage map, @@ -3149,7 +3180,7 @@ impl Pallet { Validators::::remove(who); CounterForValidators::::mutate(|x| x.saturating_dec()); VoterList::::remove(who); - debug_assert!(VoterCount::::get() == CounterForNominators::::get() + CounterForValidators::::get()); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); true } else { false diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index a584b83c539c8..7c745ac5361ec 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,14 +17,9 @@ //! Test utilities -// This module needs to exist when the `make-bags` feature is enabled so that we can generate the -// appropriate thresholds, but we don't care if it's mostly unused in that case. -#![cfg_attr(feature = "make-bags", allow(unused))] - -mod voter_bags; - -use crate::*; use crate as staking; +use crate::*; +use frame_election_provider_support::onchain; use frame_support::{ assert_ok, parameter_types, traits::{Currency, FindAuthor, Get, OnInitialize, OneSessionHandler}, @@ -39,7 +34,6 @@ use sp_runtime::{ }; use sp_staking::offence::{OffenceDetails, OnOffenceHandler}; use std::{cell::RefCell, collections::HashSet}; -use frame_election_provider_support::onchain; pub const INIT_TIMESTAMP: u64 = 30_000; pub const BLOCK_TIME: u64 = 1000; @@ -249,8 +243,11 @@ impl onchain::Config for Test { type DataProvider = Staking; } +/// Thresholds used for bags. +const THRESHOLDS: [VoteWeight; 9] = [10, 20, 30, 40, 50, 60, 1_000, 2_000, 10_000]; + parameter_types! { - pub const VoterBagThresholds: &'static [VoteWeight] = &voter_bags::THRESHOLDS; + pub const VoterBagThresholds: &'static [VoteWeight] = &THRESHOLDS; } impl Config for Test { @@ -387,14 +384,9 @@ impl ExtBuilder { } fn build(self) -> sp_io::TestExternalities { sp_tracing::try_init_simple(); - let mut storage = frame_system::GenesisConfig::default() - .build_storage::() - .unwrap(); - let balance_factor = if ExistentialDeposit::get() > 1 { - 256 - } else { - 1 - }; + + let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); + let balance_factor = if ExistentialDeposit::get() > 1 { 256 } else { 1 }; let num_validators = self.num_validators.unwrap_or(self.validator_count); // Check that the number of validators is sensible. @@ -511,6 +503,9 @@ fn check_count() { let validator_count = Validators::::iter().count() as u32; assert_eq!(nominator_count, CounterForNominators::::get()); assert_eq!(validator_count, CounterForValidators::::get()); + + let voters_count = CounterForVoters::::get(); + assert_eq!(voters_count, nominator_count + validator_count); } fn check_ledgers() { @@ -839,3 +834,14 @@ pub(crate) fn staking_events() -> Vec> { pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { (Balances::free_balance(who), Balances::reserved_balance(who)) } + +use crate::voter_bags::Bag; +/// Returns the nodes of all non-empty bags. +pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { + VoterBagThresholds::get().into_iter().filter_map(|t| { + Bag::::get(*t).map(|bag| ( + *t, + bag.iter().map(|n| n.voter().id).collect::>() + )) + }).collect::>() +} diff --git a/frame/staking/src/mock/voter_bags.rs b/frame/staking/src/mock/voter_bags.rs deleted file mode 100644 index 6177c615bc59a..0000000000000 --- a/frame/staking/src/mock/voter_bags.rs +++ /dev/null @@ -1,101 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Autogenerated voter bag thresholds. -//! -//! Generated on 2021-07-05T12:08:52.871368217+00:00 -//! for the test runtime. - -/// Existential weight for this runtime. -#[cfg(any(test, feature = "std"))] -#[allow(unused)] -pub const EXISTENTIAL_WEIGHT: u64 = 1; - -/// Constant ratio between bags for this runtime. -#[cfg(any(test, feature = "std"))] -#[allow(unused)] -pub const CONSTANT_RATIO: f64 = 2.0000000000000000; - -/// Upper thresholds delimiting the bag list. -pub const THRESHOLDS: [u64; 65] = [5, 15, 25, 50, 100, 1000, 2000, 3000] -pub const THRESHOLDS: [u64; 65] = [ - 1, - 2, - 4, - 8, - 16, - 32, - 64, - 128, - 256, - 512, - 1_024, - 2_048, - 4_096, - 8_192, - 16_384, - 32_768, - 65_536, - 131_072, - 262_144, - 524_288, - 1_048_576, - 2_097_152, - 4_194_304, - 8_388_608, - 16_777_216, - 33_554_432, - 67_108_864, - 134_217_728, - 268_435_456, - 536_870_912, - 1_073_741_824, - 2_147_483_648, - 4_294_967_296, - 8_589_934_592, - 17_179_869_184, - 34_359_738_368, - 68_719_476_736, - 137_438_953_472, - 274_877_906_944, - 549_755_813_888, - 1_099_511_627_776, - 2_199_023_255_552, - 4_398_046_511_104, - 8_796_093_022_208, - 17_592_186_044_416, - 35_184_372_088_832, - 70_368_744_177_664, - 140_737_488_355_328, - 281_474_976_710_656, - 562_949_953_421_312, - 1_125_899_906_842_624, - 2_251_799_813_685_248, - 4_503_599_627_370_496, - 9_007_199_254_740_992, - 18_014_398_509_481_984, - 36_028_797_018_963_968, - 72_057_594_037_927_936, - 144_115_188_075_855_872, - 288_230_376_151_711_744, - 576_460_752_303_423_488, - 1_152_921_504_606_846_976, - 2_305_843_009_213_693_952, - 4_611_686_018_427_387_904, - 9_223_372_036_854_775_808, - 18_446_744_073_709_551_615, -]; diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 864fcb0c35683..15a82e4858b30 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -148,16 +148,13 @@ fn basic_setup_works() { // New era is not being forced assert_eq!(Staking::force_era(), Forcing::NotForcing); - // genesis accounts must exist in the proper bags - let weight_of = Staking::weight_of_fn(); - // for these stash ids, see - // https://github.com/paritytech/substrate/ - // blob/631d4cdbcad438248c2597213918d8207d85bf6e/frame/staking/src/mock.rs#L435-L441 - for genesis_stash_account_id in [11, 21, 31, 101] { - let node = crate::voter_bags::Node::::from_id(&genesis_stash_account_id) - .expect(&format!("node was created for account {}", genesis_stash_account_id)); - assert!(!node.is_misplaced(&weight_of)); - } + // check the bags + assert_eq!(CounterForVoters::::get(), 4); + + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101])], + ); }); } @@ -3871,6 +3868,18 @@ fn on_finalize_weight_is_nonzero() { }) } +// end-to-end nodes of the voter bags operation. +mod voter_bags { + + #[test] + fn rebag_works() { + todo!() + } +} +/* +// TODO: this needs some love, retire it in favour of the one above. Use the mock data, don't make +// it complicated with data setup, use the simplest data possible, instead check multiple +// edge-cases. #[test] fn test_rebag() { use crate::{ @@ -3901,8 +3910,8 @@ fn test_rebag() { ExtBuilder::default().build_and_execute(|| { // We want to have two validators: one, `stash`, is the one we will rebag. // The other, `other_stash`, exists only so that the destination bag is not empty. - let stash = make_validator(0, 100).unwrap(); - let other_stash = make_validator(1, 300).unwrap(); + let stash = make_validator(0, 2000).unwrap(); + let other_stash = make_validator(1, 9000).unwrap(); // verify preconditions let weight_of = Staking::weight_of_fn(); @@ -3942,6 +3951,7 @@ fn test_rebag() { assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); }); } +*/ mod election_data_provider { use super::*; diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 34efce5c59f02..0141f71ef156c 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -21,12 +21,8 @@ //! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of //! voters doesn't particularly matter. -use crate::{ - AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, - VotingDataOf, slashing::SlashingSpans, -}; -use codec::{Encode, Decode}; -use frame_support::{DefaultNoBound, traits::Get}; +use codec::{Decode, Encode}; +use frame_support::{ensure, traits::Get, DefaultNoBound}; use sp_runtime::SaturatedConversion; use sp_std::{ boxed::Box, @@ -35,6 +31,11 @@ use sp_std::{ marker::PhantomData, }; +use crate::{ + slashing::SlashingSpans, AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, + VoteWeight, VoterBagFor, VotingDataOf, +}; + /// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. pub type VoterOf = Voter>; @@ -77,7 +78,7 @@ pub struct VoterList(PhantomData); impl VoterList { /// Remove all data associated with the voter list from storage. pub fn clear() { - crate::VoterCount::::kill(); + crate::CounterForVoters::::kill(); crate::VoterBagFor::::remove_all(None); crate::VoterBags::::remove_all(None); crate::VoterNodes::::remove_all(None); @@ -104,13 +105,14 @@ impl VoterList { /// Decode the length of the voter list. pub fn decode_len() -> Option { - let maybe_len = crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()); + let maybe_len = crate::CounterForVoters::::try_get().ok().map(|n| n.saturated_into()); debug_assert_eq!( maybe_len.unwrap_or_default(), crate::VoterNodes::::iter().count(), "stored length must match count of nodes", ); debug_assert_eq!( + // TODO: this case will fail in migration pre check maybe_len.unwrap_or_default() as u32, crate::CounterForNominators::::get() + crate::CounterForValidators::::get(), "voter count must be sum of validator and nominator count", @@ -175,6 +177,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); let bag = notional_bag_for::(weight); + crate::log!(debug, "inserting {:?} into bag {:?}", voter, bag); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } @@ -183,13 +186,15 @@ impl VoterList { bag.put(); } - crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_add(count) + }); count } /// Remove a voter (by id) from the voter list. pub fn remove(voter: &AccountIdOf) { - Self::remove_many(sp_std::iter::once(voter)) + Self::remove_many(sp_std::iter::once(voter)); } /// Remove many voters (by id) from the voter list. @@ -219,7 +224,9 @@ impl VoterList { bag.put(); } - crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_sub(count)); + crate::CounterForVoters::::mutate(|prev_count| { + *prev_count = prev_count.saturating_sub(count) + }); } /// Update a voter's position in the voter list. @@ -227,7 +234,7 @@ impl VoterList { /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they /// are moved into the correct bag. /// - /// Returns `true` if the voter moved. + /// Returns `Some((old_idx, new_idx))` if the voter moved, otherwise `None`. /// /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient @@ -350,6 +357,50 @@ impl VoterList { num_affected } + + /// Sanity check the voter list. + /// + /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) + /// is being used, after all other staking data (such as counter) has been updated. It checks + /// that: + /// + /// * Iterate all voters in list and make sure there are no duplicates. + /// * Iterate all voters and ensure their count is in sync with `CounterForVoters`. + /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. + /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are + /// checked per *any* update to `VoterList`. + pub(super) fn sanity_check() -> Result<(), String> { + let mut seen_in_list = BTreeSet::new(); + ensure!( + Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), + String::from("duplicate identified") + ); + + let iter_count = Self::iter().collect::>().len() as u32; + let stored_count = crate::CounterForVoters::::get(); + ensure!( + iter_count == stored_count, + format!("iter_count ({}) != voter_count ({})", iter_count, stored_count), + ); + + let validators = crate::CounterForValidators::::get(); + let nominators = crate::CounterForNominators::::get(); + ensure!( + validators + nominators == stored_count, + format!( + "validators {} + nominators {} != voters {}", + validators, nominators, stored_count + ), + ); + + let _ = T::VoterBagThresholds::get() + .into_iter() + .map(|t| Bag::::get(*t).unwrap_or_default()) + .map(|b| b.sanity_check()) + .collect::>()?; + + Ok(()) + } } /// A Bag is a doubly-linked list of voters. @@ -384,6 +435,10 @@ impl Bag { /// Get a bag by its upper vote weight or make it, appropriately initialized. pub fn get_or_make(bag_upper: VoteWeight) -> Bag { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } @@ -438,9 +493,9 @@ impl Bag { node.put(); // update the previous tail - if let Some(mut tail) = self.tail() { - tail.next = Some(id.clone()); - tail.put(); + if let Some(mut old_tail) = self.tail() { + old_tail.next = Some(id.clone()); + old_tail.put(); } // update the internal bag links @@ -458,19 +513,19 @@ impl Bag { /// the first place. Generally, use [`VoterList::remove`] instead. /// /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()` and `node.put()` after use. + /// `self.put()`, `VoterNodes::remove(voter_id)` and `VoterBagFor::remove(voter_id)` + /// to update storage for the bag and `node`. fn remove_node(&mut self, node: &Node) { - // TODO: we could merge this function here. - // node.excise(); + // Update previous node. if let Some(mut prev) = node.prev() { - prev.next = self.next.clone(); + prev.next = node.next.clone(); prev.put(); } + // Update next node. if let Some(mut next) = node.next() { - next.prev = self.prev.clone(); + next.prev = node.prev.clone(); next.put(); } - // IDEA: debug_assert! prev.next.prev == self // clear the bag head/tail pointers as necessary if self.head.as_ref() == Some(&node.voter.id) { @@ -480,6 +535,45 @@ impl Bag { self.tail = node.prev.clone(); } } + + /// Sanity check this bag. + /// + /// Should be called by the call-site, after each mutating operation on a bag. The call site of + /// this struct is always `VoterList`. + /// + /// * Ensures head has no prev. + /// * Ensures tail has no next. + /// * Ensures there are no loops, traversal from head to tail is correct. + fn sanity_check(&self) -> Result<(), String> { + ensure!( + self.head() + .map(|head| head.prev().is_none()) + // if there is no head, then there must not be a tail, meaning that the bag is + // empty. + .unwrap_or_else(|| self.tail.is_none()), + String::from("head has a prev") + ); + + ensure!( + self.tail() + .map(|tail| tail.next().is_none()) + // if there is no tail, then there must not be a head, meaning that the bag is + // empty. + .unwrap_or_else(|| self.head.is_none()), + String::from("tail has a next") + ); + + let mut seen_in_bag = BTreeSet::new(); + ensure!( + self.iter() + .map(|node| node.voter.id) + // each voter is only seen once, thus there is no cycle within a bag + .all(|voter| seen_in_bag.insert(voter)), + String::from("Duplicate found in bag.") + ); + + Ok(()) + } } /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. @@ -498,7 +592,10 @@ pub struct Node { impl Node { /// Get a node by bag idx and account id. pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { - // debug_assert!(bag_upper is in Threshold) + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { node.bag_upper = bag_upper; node @@ -563,20 +660,6 @@ impl Node { } } - /// Remove this node from the linked list. - /// - /// Modifies storage, but only modifies the adjacent nodes. Does not modify `self` or any bag. - fn excise(&self) { - if let Some(mut prev) = self.prev() { - prev.next = self.next.clone(); - prev.put(); - } - if let Some(mut next) = self.next() { - next.prev = self.prev.clone(); - next.put(); - } - } - /// `true` when this voter is in the wrong bag. pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper @@ -606,6 +689,12 @@ impl Node { let current_weight = weight_of(&self.voter.id); notional_bag_for::(current_weight) } + + #[cfg(any(test, feature = "runtime-benchmarks"))] + /// Get the underlying voter. + pub fn voter(&self) -> &Voter { + &self.voter + } } /// Fundamental information about a voter. @@ -706,7 +795,7 @@ pub mod make_bags { fn path_to_header_file() -> Option { let repo = git2::Repository::open_from_env().ok()?; let workdir = repo.workdir()?; - for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { + for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { let path = workdir.join(file_name); if path.exists() { return Some(path); @@ -853,23 +942,181 @@ pub mod make_bags { } } +// This is the highest level of abstraction provided by this module. More generic tests are here, +// among those related to `VoterList` struct. #[cfg(test)] -mod tests { - use crate::mock::{ExtBuilder, Staking, Test}; +mod voter_list { use frame_support::traits::Currency; - use substrate_test_utils::assert_eq_uvec; use super::*; + use crate::mock::*; - const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; + #[test] + fn basic_setup_works() { + // make sure ALL relevant data structures are setup correctly. + // TODO: we are not checking all of them yet. + ExtBuilder::default().build_and_execute(|| { + assert_eq!(crate::CounterForVoters::::get(), 4); + + let weight_of = Staking::weight_of_fn(); + assert_eq!(weight_of(&11), 1000); + assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); + + assert_eq!(weight_of(&21), 1000); + assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); + + assert_eq!(weight_of(&31), 1); + assert_eq!(VoterBagFor::::get(31).unwrap(), 10); + + assert_eq!(VoterBagFor::::get(41), None); // this staker is chilled! + + assert_eq!(weight_of(&101), 500); + assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); + + // iteration of the bags would yield: + assert_eq!( + VoterList::::iter().map(|n| n.voter().id).collect::>(), + vec![11, 21, 101, 31], + // ^^ note the order of insertion in genesis! + ); + + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101])], + ); + }) + } #[test] - fn voter_list_includes_genesis_accounts() { - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); - assert_eq_uvec!(GENESIS_VOTER_IDS, have_voters); - }); + fn notional_bag_for_works() { + todo!(); + } + + #[test] + fn iteration_is_semi_sorted() { + ExtBuilder::default().build_and_execute(|| { + // add some new validators to the genesis state. + bond_validator(51, 50, 2000); + bond_validator(61, 60, 2000); + + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // when + let iteration = VoterList::::iter().map(|node| node.voter.id).collect::>(); + + // then + assert_eq!(iteration, vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, // last bag. + ]); + }) + } + + /// This tests that we can `take` x voters, even if that quantity ends midway through a list. + #[test] + fn take_works() { + ExtBuilder::default().build_and_execute(|| { + // add some new validators to the genesis state. + bond_validator(51, 50, 2000); + bond_validator(61, 60, 2000); + + // given + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], + ); + + // when + let iteration = VoterList::::iter() + .map(|node| node.voter.id) + .take(4) + .collect::>(); + + // then + assert_eq!(iteration, vec![ + 51, 61, // best bag, fully iterated + 11, 21, // middle bag, partially iterated + ]); + }) + } + + #[test] + fn storage_is_cleaned_up_as_voters_are_removed() {} + + #[test] + fn insert_works() { + todo!() + } + + #[test] + fn insert_as_works() { + // insert a new one with role + // update the status of already existing one. + todo!() } + #[test] + fn remove_works() { + todo!() + } + + #[test] + fn update_position_for_works() { + // alter the genesis state to require a re-bag, then ensure this fixes it. Might be similar + // `rebag_works()` + todo!(); + } +} + +#[cfg(test)] +mod bags { + #[test] + fn get_works() { + todo!() + } + + #[test] + fn insert_works() { + todo!() + } + + #[test] + fn remove_works() { + todo!() + } +} + +#[cfg(test)] +mod voter_node { + #[test] + fn get_voting_data_works() { + todo!() + } + + #[test] + fn is_misplaced_works() { + todo!() + } +} + +// TODO: I've created simpler versions of these tests above. We can probably remove the ones below +// now. Peter was likely not very familiar with the staking mock and he came up with these rather +// complicated test setups. Please see my versions above, we can test the same properties, easily, +// without the need to alter the stakers so much. +/* +#[cfg(test)] +mod tests { + use frame_support::traits::Currency; + + use super::*; + use crate::mock::*; + + const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; + /// This tests the property that when iterating through the `VoterList`, we iterate from higher /// bags to lower. #[test] @@ -888,8 +1135,7 @@ mod tests { ExtBuilder::default().validator_pool(true).build_and_execute(|| { // initialize the voters' deposits - let existential_deposit = ::Currency::minimum_balance(); - let mut balance = existential_deposit + 1; + let mut balance = 10; for voter_id in voters.iter().rev() { ::Currency::make_free_balance_be(voter_id, balance); let controller = Staking::bonded(voter_id).unwrap(); @@ -899,11 +1145,138 @@ mod tests { Staking::update_ledger(&controller, &ledger); Staking::do_rebag(voter_id); - balance *= 2; + // Increase balance to the next threshold. + balance += 10; } let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); assert_eq!(voters, have_voters); }); } + + /// This tests that we can `take` x voters, even if that quantity ends midway through a list. + #[test] + fn take_works() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // initialize the voters' deposits + let mut balance = 0; // This will be 10 on the first loop iteration because 0 % 3 == 0 + for (idx, voter_id) in GENESIS_VOTER_IDS.iter().enumerate() { + if idx % 3 == 0 { + // This increases the balance by 10, which is the amount each threshold + // increases by. Thus this will increase the balance by 1 bag. + // + // This will create 2 bags, the lower threshold bag having + // 3 voters with balance 10, and the higher threshold bag having + // 2 voters with balance 20. + balance += 10; + } + + ::Currency::make_free_balance_be(voter_id, balance); + let controller = Staking::bonded(voter_id).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = balance; + ledger.active = balance; + Staking::update_ledger(&controller, &ledger); + Staking::do_rebag(voter_id); + } + + let bag_thresh10 = Bag::::get(10) + .unwrap() + .iter() + .map(|node| node.voter.id) + .collect::>(); + assert_eq!(bag_thresh10, vec![11, 21, 31]); + + let bag_thresh20 = Bag::::get(20) + .unwrap() + .iter() + .map(|node| node.voter.id) + .collect::>(); + assert_eq!(bag_thresh20, vec![41, 101]); + + let voters: Vec<_> = VoterList::::iter() + // take 4/5 from [41, 101],[11, 21, 31], demonstrating that we can do a + // take that stops mid bag. + .take(4) + .map(|node| node.voter.id) + .collect(); + + assert_eq!(voters, vec![41, 101, 11, 21]); + }); + } + + #[test] + fn storage_is_cleaned_up_as_voters_are_removed() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // Initialize voters deposits so there are 5 bags with one voter each. + let mut balance = 10; + for voter_id in GENESIS_VOTER_IDS.iter() { + ::Currency::make_free_balance_be(voter_id, balance); + let controller = Staking::bonded(voter_id).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = balance; + ledger.active = balance; + Staking::update_ledger(&controller, &ledger); + Staking::do_rebag(voter_id); + + // Increase balance to the next threshold. + balance += 10; + } + + let voter_list_storage_items_eq = |mut v: Vec| { + v.sort(); + let mut voters: Vec<_> = + VoterList::::iter().map(|node| node.voter.id).collect(); + voters.sort(); + assert_eq!(voters, v); + + let mut nodes: Vec<_> = + ::VoterNodes::iter_keys().collect(); + nodes.sort(); + assert_eq!(nodes, v); + + let mut flat_bags: Vec<_> = ::VoterBags::iter() + // We always get the bag with the Bag getter because the bag_upper + // is only initialized in the getter. + .flat_map(|(key, _bag)| Bag::::get(key).unwrap().iter()) + .map(|node| node.voter.id) + .collect(); + flat_bags.sort(); + assert_eq!(flat_bags, v); + + let mut bags_for: Vec<_> = + ::VoterBagFor::iter_keys().collect(); + bags_for.sort(); + assert_eq!(bags_for, v); + }; + + let genesis_voters = vec![101, 41, 31, 21, 11]; + voter_list_storage_items_eq(genesis_voters); + assert_eq!(::CounterForVoters::get(), 5); + + // Remove 1 voter, + VoterList::::remove(&101); + let remaining_voters = vec![41, 31, 21, 11]; + // and assert they have been cleaned up. + voter_list_storage_items_eq(remaining_voters.clone()); + assert_eq!(::CounterForVoters::get(), 4); + + // Now remove the remaining voters so we have 0 left, + remaining_voters.iter().for_each(|v| VoterList::::remove(v)); + // and assert all of them have been cleaned up. + voter_list_storage_items_eq(vec![]); + assert_eq!(::CounterForVoters::get(), 0); + + + // TODO bags do not get cleaned up from storages + // - is this ok? I assume its ok if this is not cleaned just because voters are removed + // but it should be cleaned up if we migrate thresholds + assert_eq!(::VoterBags::iter().collect::>().len(), 6); + // and the voter list has no one in it. + assert_eq!(VoterList::::iter().collect::>().len(), 0); + assert_eq!(::VoterBagFor::iter().collect::>().len(), 0); + assert_eq!(::VoterNodes::iter().collect::>().len(), 0); + }); + } } +*/ From 48ccfc9df1e95675f0cbc7719cf46533befe5c6d Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 21 Jul 2021 17:21:43 -0700 Subject: [PATCH 72/82] merge fallout --- bin/node/runtime/voter-bags/Cargo.toml | 6 +++--- frame/staking/Cargo.toml | 11 ++++++----- frame/staking/src/voter_bags.rs | 9 ++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/node/runtime/voter-bags/Cargo.toml b/bin/node/runtime/voter-bags/Cargo.toml index 38f58ba9d4bd0..55b9dca859db8 100644 --- a/bin/node/runtime/voter-bags/Cargo.toml +++ b/bin/node/runtime/voter-bags/Cargo.toml @@ -10,7 +10,7 @@ description = "Voter Bag generation script for pallet-staking and node-runtime." readme = "README.md" [dependencies] -node-runtime = { version = "2.0.0", path = ".." } -pallet-staking = { version = "3.0.0", path = "../../../../frame/staking", features = ["make-bags"] } -sp-io = { version = "3.0.0", path = "../../../../primitives/io" } +node-runtime = { version = "3.0.0-dev", path = ".." } +pallet-staking = { version = "4.0.0-dev", path = "../../../../frame/staking", features = ["make-bags"] } +sp-io = { version = "4.0.0-dev", path = "../../../../primitives/io" } structopt = "0.3.21" diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 22c5fa9ef6ecd..bfe2e04475452 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -26,11 +26,11 @@ pallet-session = { version = "4.0.0-dev", default-features = false, features = [ pallet-authorship = { version = "4.0.0-dev", default-features = false, path = "../authorship" } sp-application-crypto = { version = "4.0.0-dev", default-features = false, path = "../../primitives/application-crypto" } frame-election-provider-support = { version = "4.0.0-dev", default-features = false, path = "../election-provider-support" } -sp-core = { version = "3.0.0", path = "../../primitives/core", optional = true } -pallet-timestamp = { version = "3.0.0", path = "../timestamp", optional = true } -pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve", optional = true } -pallet-balances = { version = "3.0.0", path = "../balances", optional = true } -sp-tracing = { version = "3.0.0", path = "../../primitives/tracing", optional = true } +sp-core = { version = "4.0.0-dev", path = "../../primitives/core", optional = true } +pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp", optional = true } +pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve", optional = true } +pallet-balances = { version = "4.0.0-dev", path = "../balances", optional = true } +sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing", optional = true } log = { version = "0.4.14", default-features = false } paste = "1.0" @@ -47,6 +47,7 @@ num-format = { version = "0.4.0", optional = true } sp-storage = { version = "4.0.0-dev", path = "../../primitives/storage" } sp-tracing = { version = "4.0.0-dev", path = "../../primitives/tracing" } sp-core = { version = "4.0.0-dev", path = "../../primitives/core" } +sp-npos-elections = { version = "4.0.0-dev", path = "../../primitives/npos-elections", features = ["mocks"] } pallet-balances = { version = "4.0.0-dev", path = "../balances" } pallet-timestamp = { version = "4.0.0-dev", path = "../timestamp" } pallet-staking-reward-curve = { version = "4.0.0-dev", path = "../staking/reward-curve" } diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 0141f71ef156c..edc72e7be6865 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -369,11 +369,11 @@ impl VoterList { /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `VoterList`. - pub(super) fn sanity_check() -> Result<(), String> { + pub(super) fn sanity_check() -> Result<(), std::string::String> { let mut seen_in_list = BTreeSet::new(); ensure!( Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), - String::from("duplicate identified") + String::from("duplicate identified"), ); let iter_count = Self::iter().collect::>().len() as u32; @@ -544,7 +544,7 @@ impl Bag { /// * Ensures head has no prev. /// * Ensures tail has no next. /// * Ensures there are no loops, traversal from head to tail is correct. - fn sanity_check(&self) -> Result<(), String> { + fn sanity_check(&self) -> Result<(), std::string::String> { ensure!( self.head() .map(|head| head.prev().is_none()) @@ -698,8 +698,7 @@ impl Node { } /// Fundamental information about a voter. -#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr(feature = "std", derive(Debug))] +#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord, sp_runtime::RuntimeDebug)] pub struct Voter { /// Account Id of this voter pub id: AccountId, From 1c56016e3e016d103c52d5abd2003e594cf665c0 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 21 Jul 2021 17:54:27 -0700 Subject: [PATCH 73/82] Run cargo +nightly fmt --- bin/node/runtime/src/voter_bags.rs | 378 +++++++++--------- bin/node/runtime/voter-bags/src/main.rs | 12 +- .../election-provider-support/src/onchain.rs | 6 +- frame/staking/src/lib.rs | 47 +-- frame/staking/src/mock.rs | 13 +- frame/staking/src/tests.rs | 7 +- frame/staking/src/voter_bags.rs | 78 ++-- primitives/npos-elections/src/lib.rs | 5 +- 8 files changed, 265 insertions(+), 281 deletions(-) diff --git a/bin/node/runtime/src/voter_bags.rs b/bin/node/runtime/src/voter_bags.rs index f8f6201e94dc3..c4c731a58badc 100644 --- a/bin/node/runtime/src/voter_bags.rs +++ b/bin/node/runtime/src/voter_bags.rs @@ -32,195 +32,195 @@ pub const CONSTANT_RATIO: f64 = 1.0628253590743408; /// Upper thresholds delimiting the bag list. pub const THRESHOLDS: [u64; 200] = [ - 100_000_000_000_000, - 106_282_535_907_434, - 112_959_774_389_150, - 120_056_512_776_105, - 127_599_106_300_477, - 135_615_565_971_369, - 144_135_662_599_590, - 153_191_037_357_827, - 162_815_319_286_803, - 173_044_250_183_800, - 183_915_817_337_347, - 195_470_394_601_017, - 207_750_892_330_229, - 220_802_916_738_890, - 234_674_939_267_673, - 249_418_476_592_914, - 265_088_281_944_639, - 281_742_548_444_211, - 299_443_125_216_738, - 318_255_747_080_822, - 338_250_278_668_647, - 359_500_973_883_001, - 382_086_751_654_776, - 406_091_489_025_036, - 431_604_332_640_068, - 458_720_029_816_222, - 487_539_280_404_019, - 518_169_110_758_247, - 550_723_271_202_866, - 585_322_658_466_782, - 622_095_764_659_305, - 661_179_154_452_653, - 702_717_972_243_610, - 746_866_481_177_808, - 793_788_636_038_393, - 843_658_692_126_636, - 896_661_852_395_681, - 952_994_955_240_703, - 1_012_867_205_499_736, - 1_076_500_951_379_881, - 1_144_132_510_194_192, - 1_216_013_045_975_769, - 1_292_409_502_228_280, - 1_373_605_593_276_862, - 1_459_902_857_901_004, - 1_551_621_779_162_291, - 1_649_102_974_585_730, - 1_752_708_461_114_642, - 1_862_822_999_536_805, - 1_979_855_523_374_646, - 2_104_240_657_545_975, - 2_236_440_332_435_128, - 2_376_945_499_368_703, - 2_526_277_953_866_680, - 2_684_992_273_439_945, - 2_853_677_877_130_641, - 3_032_961_214_443_876, - 3_223_508_091_799_862, - 3_426_026_145_146_232, - 3_641_267_467_913_124, - 3_870_031_404_070_482, - 4_113_167_516_660_186, - 4_371_578_742_827_277, - 4_646_224_747_067_156, - 4_938_125_485_141_739, - 5_248_364_991_899_922, - 5_578_095_407_069_235, - 5_928_541_253_969_291, - 6_301_003_987_036_955, - 6_696_866_825_051_405, - 7_117_599_888_008_300, - 7_564_765_656_719_910, - 8_040_024_775_416_580, - 8_545_142_218_898_723, - 9_081_993_847_142_344, - 9_652_573_371_700_016, - 10_258_999_759_768_490, - 10_903_525_103_419_522, - 11_588_542_983_217_942, - 12_316_597_357_287_042, - 13_090_392_008_832_678, - 13_912_800_587_211_472, - 14_786_877_279_832_732, - 15_715_868_154_526_436, - 16_703_223_214_499_558, - 17_752_609_210_649_358, - 18_867_923_258_814_856, - 20_053_307_312_537_008, - 21_313_163_545_075_252, - 22_652_170_697_804_756, - 24_075_301_455_707_600, - 25_587_840_914_485_432, - 27_195_406_207_875_088, - 28_903_967_368_057_400, - 30_719_869_496_628_636, - 32_649_856_328_471_220, - 34_701_095_276_033_064, - 36_881_204_047_022_752, - 39_198_278_934_370_992, - 41_660_924_883_519_016, - 44_278_287_448_695_240, - 47_060_086_756_856_400, - 50_016_653_605_425_536, - 53_158_967_827_883_320, - 56_498_699_069_691_424, - 60_048_250_125_977_912, - 63_820_803_001_928_304, - 67_830_367_866_937_216, - 72_091_835_084_322_176, - 76_621_030_509_822_880, - 81_434_774_264_248_528, - 86_550_943_198_537_824, - 91_988_537_283_208_848, - 97_767_750_168_749_840, - 103_910_044_178_992_000, - 110_438_230_015_967_792, - 117_376_551_472_255_616, - 124_750_775_465_407_920, - 132_588_287_728_824_640, - 140_918_194_514_440_064, - 149_771_430_684_917_568, - 159_180_874_596_775_264, - 169_181_470_201_085_280, - 179_810_356_815_193_344, - 191_107_007_047_393_216, - 203_113_373_386_768_288, - 215_874_044_002_592_672, - 229_436_408_331_885_600, - 243_850_833_070_063_392, - 259_170_849_218_267_264, - 275_453_350_882_006_752, - 292_758_806_559_399_232, - 311_151_483_703_668_992, - 330_699_687_393_865_920, - 351_476_014_000_157_824, - 373_557_620_785_735_808, - 397_026_512_446_556_096, - 421_969_845_653_044_224, - 448_480_252_724_740_928, - 476_656_185_639_923_904, - 506_602_281_657_757_760, - 538_429_751_910_786_752, - 572_256_794_410_890_176, - 608_209_033_002_485_632, - 646_419_983_893_124_352, - 687_031_551_494_039_552, - 730_194_555_412_054_016, - 776_069_290_549_944_960, - 824_826_122_395_314_176, - 876_646_119_708_695_936, - 931_721_726_960_522_368, - 990_257_479_014_182_144, - 1_052_470_760_709_299_712, - 1_118_592_614_166_106_112, - 1_188_868_596_808_997_376, - 1_263_559_693_295_730_432, - 1_342_943_284_738_898_688, - 1_427_314_178_819_094_784, - 1_516_985_704_615_302_400, - 1_612_290_876_218_400_768, - 1_713_583_629_449_105_408, - 1_821_240_136_273_157_632, - 1_935_660_201_795_120_128, - 2_057_268_749_018_809_600, - 2_186_517_396_888_336_384, - 2_323_886_137_470_138_880, - 2_469_885_118_504_583_168, - 2_625_056_537_947_004_416, - 2_789_976_657_533_970_944, - 2_965_257_942_852_572_160, - 3_151_551_337_860_326_400, - 3_349_548_682_302_620_672, - 3_559_985_281_005_267_968, - 3_783_642_634_583_792_128, - 4_021_351_341_710_503_936, - 4_273_994_183_717_548_544, - 4_542_509_402_991_247_872, - 4_827_894_187_332_742_144, - 5_131_208_373_224_844_288, - 5_453_578_381_757_959_168, - 5_796_201_401_831_965_696, - 6_160_349_836_169_256_960, - 6_547_376_026_650_146_816, - 6_958_717_276_519_173_120, - 7_395_901_188_113_309_696, - 7_860_551_335_934_872_576, - 8_354_393_296_137_270_272, - 8_879_261_054_815_360_000, - 9_437_103_818_898_946_048, + 100_000_000_000_000, + 106_282_535_907_434, + 112_959_774_389_150, + 120_056_512_776_105, + 127_599_106_300_477, + 135_615_565_971_369, + 144_135_662_599_590, + 153_191_037_357_827, + 162_815_319_286_803, + 173_044_250_183_800, + 183_915_817_337_347, + 195_470_394_601_017, + 207_750_892_330_229, + 220_802_916_738_890, + 234_674_939_267_673, + 249_418_476_592_914, + 265_088_281_944_639, + 281_742_548_444_211, + 299_443_125_216_738, + 318_255_747_080_822, + 338_250_278_668_647, + 359_500_973_883_001, + 382_086_751_654_776, + 406_091_489_025_036, + 431_604_332_640_068, + 458_720_029_816_222, + 487_539_280_404_019, + 518_169_110_758_247, + 550_723_271_202_866, + 585_322_658_466_782, + 622_095_764_659_305, + 661_179_154_452_653, + 702_717_972_243_610, + 746_866_481_177_808, + 793_788_636_038_393, + 843_658_692_126_636, + 896_661_852_395_681, + 952_994_955_240_703, + 1_012_867_205_499_736, + 1_076_500_951_379_881, + 1_144_132_510_194_192, + 1_216_013_045_975_769, + 1_292_409_502_228_280, + 1_373_605_593_276_862, + 1_459_902_857_901_004, + 1_551_621_779_162_291, + 1_649_102_974_585_730, + 1_752_708_461_114_642, + 1_862_822_999_536_805, + 1_979_855_523_374_646, + 2_104_240_657_545_975, + 2_236_440_332_435_128, + 2_376_945_499_368_703, + 2_526_277_953_866_680, + 2_684_992_273_439_945, + 2_853_677_877_130_641, + 3_032_961_214_443_876, + 3_223_508_091_799_862, + 3_426_026_145_146_232, + 3_641_267_467_913_124, + 3_870_031_404_070_482, + 4_113_167_516_660_186, + 4_371_578_742_827_277, + 4_646_224_747_067_156, + 4_938_125_485_141_739, + 5_248_364_991_899_922, + 5_578_095_407_069_235, + 5_928_541_253_969_291, + 6_301_003_987_036_955, + 6_696_866_825_051_405, + 7_117_599_888_008_300, + 7_564_765_656_719_910, + 8_040_024_775_416_580, + 8_545_142_218_898_723, + 9_081_993_847_142_344, + 9_652_573_371_700_016, + 10_258_999_759_768_490, + 10_903_525_103_419_522, + 11_588_542_983_217_942, + 12_316_597_357_287_042, + 13_090_392_008_832_678, + 13_912_800_587_211_472, + 14_786_877_279_832_732, + 15_715_868_154_526_436, + 16_703_223_214_499_558, + 17_752_609_210_649_358, + 18_867_923_258_814_856, + 20_053_307_312_537_008, + 21_313_163_545_075_252, + 22_652_170_697_804_756, + 24_075_301_455_707_600, + 25_587_840_914_485_432, + 27_195_406_207_875_088, + 28_903_967_368_057_400, + 30_719_869_496_628_636, + 32_649_856_328_471_220, + 34_701_095_276_033_064, + 36_881_204_047_022_752, + 39_198_278_934_370_992, + 41_660_924_883_519_016, + 44_278_287_448_695_240, + 47_060_086_756_856_400, + 50_016_653_605_425_536, + 53_158_967_827_883_320, + 56_498_699_069_691_424, + 60_048_250_125_977_912, + 63_820_803_001_928_304, + 67_830_367_866_937_216, + 72_091_835_084_322_176, + 76_621_030_509_822_880, + 81_434_774_264_248_528, + 86_550_943_198_537_824, + 91_988_537_283_208_848, + 97_767_750_168_749_840, + 103_910_044_178_992_000, + 110_438_230_015_967_792, + 117_376_551_472_255_616, + 124_750_775_465_407_920, + 132_588_287_728_824_640, + 140_918_194_514_440_064, + 149_771_430_684_917_568, + 159_180_874_596_775_264, + 169_181_470_201_085_280, + 179_810_356_815_193_344, + 191_107_007_047_393_216, + 203_113_373_386_768_288, + 215_874_044_002_592_672, + 229_436_408_331_885_600, + 243_850_833_070_063_392, + 259_170_849_218_267_264, + 275_453_350_882_006_752, + 292_758_806_559_399_232, + 311_151_483_703_668_992, + 330_699_687_393_865_920, + 351_476_014_000_157_824, + 373_557_620_785_735_808, + 397_026_512_446_556_096, + 421_969_845_653_044_224, + 448_480_252_724_740_928, + 476_656_185_639_923_904, + 506_602_281_657_757_760, + 538_429_751_910_786_752, + 572_256_794_410_890_176, + 608_209_033_002_485_632, + 646_419_983_893_124_352, + 687_031_551_494_039_552, + 730_194_555_412_054_016, + 776_069_290_549_944_960, + 824_826_122_395_314_176, + 876_646_119_708_695_936, + 931_721_726_960_522_368, + 990_257_479_014_182_144, + 1_052_470_760_709_299_712, + 1_118_592_614_166_106_112, + 1_188_868_596_808_997_376, + 1_263_559_693_295_730_432, + 1_342_943_284_738_898_688, + 1_427_314_178_819_094_784, + 1_516_985_704_615_302_400, + 1_612_290_876_218_400_768, + 1_713_583_629_449_105_408, + 1_821_240_136_273_157_632, + 1_935_660_201_795_120_128, + 2_057_268_749_018_809_600, + 2_186_517_396_888_336_384, + 2_323_886_137_470_138_880, + 2_469_885_118_504_583_168, + 2_625_056_537_947_004_416, + 2_789_976_657_533_970_944, + 2_965_257_942_852_572_160, + 3_151_551_337_860_326_400, + 3_349_548_682_302_620_672, + 3_559_985_281_005_267_968, + 3_783_642_634_583_792_128, + 4_021_351_341_710_503_936, + 4_273_994_183_717_548_544, + 4_542_509_402_991_247_872, + 4_827_894_187_332_742_144, + 5_131_208_373_224_844_288, + 5_453_578_381_757_959_168, + 5_796_201_401_831_965_696, + 6_160_349_836_169_256_960, + 6_547_376_026_650_146_816, + 6_958_717_276_519_173_120, + 7_395_901_188_113_309_696, + 7_860_551_335_934_872_576, + 8_354_393_296_137_270_272, + 8_879_261_054_815_360_000, + 9_437_103_818_898_946_048, 10_029_993_254_943_105_024, 10_660_131_182_698_121_216, 11_329_857_752_030_707_712, diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 9ae50914d6927..2a2be8297eb22 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,12 +17,11 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::voter_bags::make_bags::generate_thresholds_module; -use pallet_staking::mock::Test; +use pallet_staking::{mock::Test, voter_bags::make_bags::generate_thresholds_module}; use std::path::{Path, PathBuf}; -use structopt::{StructOpt, clap::arg_enum}; +use structopt::{clap::arg_enum, StructOpt}; -arg_enum!{ +arg_enum! { #[derive(Debug)] enum Runtime { Node, @@ -42,10 +41,7 @@ impl Runtime { #[derive(Debug, StructOpt)] struct Opt { /// How many bags to generate. - #[structopt( - long, - default_value = "200", - )] + #[structopt(long, default_value = "200")] n_bags: usize, /// Which runtime to generate. diff --git a/frame/election-provider-support/src/onchain.rs b/frame/election-provider-support/src/onchain.rs index c3ca7580f66e7..6709fe33d9b95 100644 --- a/frame/election-provider-support/src/onchain.rs +++ b/frame/election-provider-support/src/onchain.rs @@ -77,8 +77,10 @@ impl ElectionProvider for OnChainSequen let (desired_targets, _) = Self::DataProvider::desired_targets().map_err(Error::DataProvider)?; - let stake_map: BTreeMap = -voters.iter().map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)).collect(); + let stake_map: BTreeMap = voters + .iter() + .map(|(validator, vote_weight, _)| (validator.clone(), *vote_weight)) + .collect(); let stake_of = |w: &T::AccountId| -> VoteWeight { stake_map.get(w).cloned().unwrap_or_default() }; diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index a617cabfe78cc..dbcf66292308f 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -276,10 +276,10 @@ #[cfg(any(test, feature = "make-bags"))] pub mod mock; -#[cfg(test)] -mod tests; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod testing_utils; +#[cfg(test)] +mod tests; pub mod inflation; pub mod slashing; @@ -303,19 +303,19 @@ use frame_system::{ensure_root, ensure_signed, offchain::SendTransactionTypes, p pub use pallet::*; use pallet_session::historical; use sp_runtime::{ - DispatchError, Perbill, Percent, RuntimeDebug, curve::PiecewiseLinear, traits::{ AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, StaticLookup, Zero, }, + DispatchError, Perbill, Percent, RuntimeDebug, }; use sp_staking::{ offence::{Offence, OffenceDetails, OffenceError, OnOffenceHandler, ReportOffence}, SessionIndex, }; -use voter_bags::{VoterList, VoterType}; use sp_std::{collections::btree_map::BTreeMap, convert::From, prelude::*, result}; +use voter_bags::{VoterList, VoterType}; pub use weights::WeightInfo; const STAKING_ID: LockIdentifier = *b"staking "; @@ -763,7 +763,10 @@ pub mod migrations { pub fn pre_migrate() -> Result<(), &'static str> { ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); - ensure!(VoterList::::decode_len().unwrap_or_default() == 0, "voter list already exists"); + ensure!( + VoterList::::decode_len().unwrap_or_default() == 0, + "voter list already exists" + ); Ok(()) } @@ -783,7 +786,8 @@ pub mod migrations { T::WeightInfo::regenerate( CounterForValidators::::get(), CounterForNominators::::get(), - ).saturating_add(T::DbWeight::get().reads(2)) + ) + .saturating_add(T::DbWeight::get().reads(2)) } } @@ -1305,7 +1309,6 @@ pub mod pallet { #[pallet::storage] pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; - // The next storage items collectively comprise the voter bags: a composite data structure // designed to allow efficient iteration of the top N voters by stake, mostly. See // `mod voter_bags` for details. @@ -1321,16 +1324,14 @@ pub mod pallet { /// This may not be the appropriate bag for the voter's weight if they have been rewarded or /// slashed. #[pallet::storage] - pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; + pub(crate) type VoterBagFor = + StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] - pub(crate) type VoterBags = StorageMap< - _, - Twox64Concat, VoteWeight, - voter_bags::Bag, - >; + pub(crate) type VoterBags = + StorageMap<_, Twox64Concat, VoteWeight, voter_bags::Bag>; /// Voter nodes store links forward and back within their respective bags, the stash id, and /// whether the voter is a validator or nominator. @@ -1338,11 +1339,8 @@ pub mod pallet { /// There is nothing in this map directly identifying to which bag a particular node belongs. /// However, the `Node` data structure has helpers which can provide that information. #[pallet::storage] - pub(crate) type VoterNodes = StorageMap< - _, - Twox64Concat, AccountIdOf, - voter_bags::Node, - >; + pub(crate) type VoterNodes = + StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::Node>; // End of voter bags data. @@ -1421,7 +1419,7 @@ pub mod pallet { // TODO: later on, fix all the tests that trigger these warnings, and // make these assertions. Genesis stakers should all be correct! log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue; + continue } match status { StakerStatus::Validator => { @@ -1431,7 +1429,7 @@ pub mod pallet { ) { log!(warn, "failed to validate staker at genesis: {:?}.", why); } else { - num_voters +=1 ; + num_voters += 1; } }, StakerStatus::Nominator(votes) => { @@ -1443,8 +1441,8 @@ pub mod pallet { } else { num_voters += 1; } - } - _ => () + }, + _ => (), }; } @@ -2502,10 +2500,7 @@ pub mod pallet { /// /// Anyone can call this function about any stash. #[pallet::weight(T::WeightInfo::rebag())] - pub fn rebag( - origin: OriginFor, - stash: AccountIdOf, - ) -> DispatchResult { + pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { ensure_signed(origin)?; Pallet::::do_rebag(&stash); Ok(()) diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 1f9a4c7a73832..57f87a036516a 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -825,10 +825,11 @@ pub(crate) fn balances(who: &AccountId) -> (Balance, Balance) { use crate::voter_bags::Bag; /// Returns the nodes of all non-empty bags. pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { - VoterBagThresholds::get().into_iter().filter_map(|t| { - Bag::::get(*t).map(|bag| ( - *t, - bag.iter().map(|n| n.voter().id).collect::>() - )) - }).collect::>() + VoterBagThresholds::get() + .into_iter() + .filter_map(|t| { + Bag::::get(*t) + .map(|bag| (*t, bag.iter().map(|n| n.voter().id).collect::>())) + }) + .collect::>() } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 780f39736621d..4724d2cd89919 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -25,6 +25,7 @@ use frame_support::{ weights::{extract_actual_weight, GetDispatchInfo}, }; use mock::*; +use pallet_balances::Error as BalancesError; use sp_npos_elections::supports_eq_unordered; use sp_runtime::{ assert_eq_error_rate, @@ -32,7 +33,6 @@ use sp_runtime::{ }; use sp_staking::offence::OffenceDetails; use substrate_test_utils::assert_eq_uvec; -use pallet_balances::Error as BalancesError; #[test] fn force_unstake_works() { @@ -174,10 +174,7 @@ fn basic_setup_works() { // check the bags assert_eq!(CounterForVoters::::get(), 4); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101])], - ); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); }); } diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index edc72e7be6865..4917e3446d884 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -97,10 +97,7 @@ impl VoterList { let validators_iter = Validators::::iter().map(|(id, _)| Voter::validator(id)); let weight_of = Pallet::::weight_of_fn(); - Self::insert_many( - nominators_iter.chain(validators_iter), - weight_of, - ) + Self::insert_many(nominators_iter.chain(validators_iter), weight_of) } /// Decode the length of the voter list. @@ -140,9 +137,7 @@ impl VoterList { // otherwise, insert it here. Box::new(iter.chain(iter::once(VoteWeight::MAX)).rev()) }; - iter - .filter_map(Bag::get) - .flat_map(|bag| bag.iter()) + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } /// Insert a new voter into the appropriate bag in the voter list. @@ -212,7 +207,9 @@ impl VoterList { count += 1; // clear the bag head/tail pointers as necessary - let bag = bags.entry(node.bag_upper).or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); bag.remove_node(&node); // now get rid of the node itself @@ -470,12 +467,7 @@ impl Bag { /// Storage note: this modifies storage, but only for the nodes. You still need to call /// `self.put()` after use. fn insert(&mut self, voter: VoterOf) { - self.insert_node(Node:: { - voter, - prev: None, - next: None, - bag_upper: self.bag_upper, - }); + self.insert_node(Node:: { voter, prev: None, next: None, bag_upper: self.bag_upper }); } /// Insert a voter node into this bag. @@ -656,7 +648,7 @@ impl Node { (!targets.is_empty()) .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) - } + }, } } @@ -708,17 +700,11 @@ pub struct Voter { impl Voter { pub fn nominator(id: AccountId) -> Self { - Self { - id, - voter_type: VoterType::Nominator, - } + Self { id, voter_type: VoterType::Nominator } } pub fn validator(id: AccountId) -> Self { - Self { - id, - voter_type: VoterType::Validator, - } + Self { id, voter_type: VoterType::Validator } } } @@ -782,10 +768,13 @@ pub fn existential_weight() -> VoteWeight { /// ``` #[cfg(feature = "make-bags")] pub mod make_bags { - use crate::{Config, voter_bags::existential_weight}; + use crate::{voter_bags::existential_weight, Config}; use frame_election_provider_support::VoteWeight; use frame_support::traits::Get; - use std::{io::Write, path::{Path, PathBuf}}; + use std::{ + io::Write, + path::{Path, PathBuf}, + }; /// Return the path to a header file used in this repository if is exists. /// @@ -797,7 +786,7 @@ pub mod make_bags { for file_name in &["HEADER-APACHE2", "HEADER-GPL3", "HEADER", "file_header.txt"] { let path = workdir.join(file_name); if path.exists() { - return Some(path); + return Some(path) } } None @@ -945,9 +934,9 @@ pub mod make_bags { // among those related to `VoterList` struct. #[cfg(test)] mod voter_list { - use frame_support::traits::Currency; use super::*; use crate::mock::*; + use frame_support::traits::Currency; #[test] fn basic_setup_works() { @@ -978,10 +967,7 @@ mod voter_list { // ^^ note the order of insertion in genesis! ); - assert_eq!( - get_bags(), - vec![(10, vec![31]), (1000, vec![11, 21, 101])], - ); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); }) } @@ -1007,11 +993,14 @@ mod voter_list { let iteration = VoterList::::iter().map(|node| node.voter.id).collect::>(); // then - assert_eq!(iteration, vec![ - 51, 61, // best bag - 11, 21, 101, // middle bag - 31, // last bag. - ]); + assert_eq!( + iteration, + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, // last bag. + ] + ); }) } @@ -1030,16 +1019,17 @@ mod voter_list { ); // when - let iteration = VoterList::::iter() - .map(|node| node.voter.id) - .take(4) - .collect::>(); + let iteration = + VoterList::::iter().map(|node| node.voter.id).take(4).collect::>(); // then - assert_eq!(iteration, vec![ - 51, 61, // best bag, fully iterated - 11, 21, // middle bag, partially iterated - ]); + assert_eq!( + iteration, + vec![ + 51, 61, // best bag, fully iterated + 11, 21, // middle bag, partially iterated + ] + ); }) } diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index cdcb90d4df644..ad2b3229a0881 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -499,7 +499,10 @@ impl Support { pub type Supports = Vec<(A, Support)>; #[cfg(feature = "mocks")] -pub fn supports_eq_unordered(a: &Supports, b: &Supports) -> bool { +pub fn supports_eq_unordered( + a: &Supports, + b: &Supports, +) -> bool { let map: BTreeMap<_, _> = a.iter().cloned().collect(); b.iter().all(|(id, b_support)| { let a_support = match map.get(id) { From 1f02cb4dc1389a9ad6886faaf58b3b3597934e14 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 22 Jul 2021 11:29:20 +0200 Subject: [PATCH 74/82] Fix a bunch of stuff, remove not needed runtime arg of make-bags --- bin/node/runtime/voter-bags/src/main.rs | 38 +++++-------------------- frame/staking/src/lib.rs | 8 ++++-- frame/staking/src/voter_bags.rs | 26 ++++++----------- 3 files changed, 20 insertions(+), 52 deletions(-) diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 2a2be8297eb22..1340285c29a1a 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,26 +17,9 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::{mock::Test, voter_bags::make_bags::generate_thresholds_module}; -use std::path::{Path, PathBuf}; -use structopt::{clap::arg_enum, StructOpt}; - -arg_enum! { - #[derive(Debug)] - enum Runtime { - Node, - StakingMock, - } -} - -impl Runtime { - fn generate_thresholds(&self) -> Box Result<(), std::io::Error>> { - match self { - Runtime::Node => Box::new(generate_thresholds_module::), - Runtime::StakingMock => Box::new(generate_thresholds_module::), - } - } -} +use pallet_staking::{voter_bags::make_bags::generate_thresholds_module}; +use std::path::PathBuf; +use structopt::StructOpt; #[derive(Debug, StructOpt)] struct Opt { @@ -44,21 +27,14 @@ struct Opt { #[structopt(long, default_value = "200")] n_bags: usize, - /// Which runtime to generate. - #[structopt( - long, - case_insensitive = true, - default_value = "Node", - possible_values = &Runtime::variants(), - )] - runtime: Runtime, - /// Where to write the output. output: PathBuf, } fn main() -> Result<(), std::io::Error> { - let Opt { n_bags, output, runtime } = Opt::from_args(); + let Opt { n_bags, output } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| runtime.generate_thresholds()(n_bags, &output)) + ext.execute_with(|| { + generate_thresholds_module::(n_bags, &output) + }) } diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index dbcf66292308f..1350582d99fd8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -271,13 +271,15 @@ //! - [Session](../pallet_session/index.html): Used to manage sessions. Also, a list of new //! validators is stored in the Session pallet's `Validators` at the end of each era. -#![recursion_limit = "128"] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(any(test, feature = "make-bags"))] -pub mod mock; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod testing_utils; +#[cfg(any(feature = "runtime-benchmarks", test))] +pub mod benchmarking; + +#[cfg(test)] +pub(crate) mod mock; #[cfg(test)] mod tests; diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 4917e3446d884..687f24f539178 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -366,29 +366,20 @@ impl VoterList { /// * Ensure `CounterForVoters` is `CounterForValidators + CounterForNominators`. /// * Sanity-checks all bags. This will cascade down all the checks and makes sure all bags are /// checked per *any* update to `VoterList`. - pub(super) fn sanity_check() -> Result<(), std::string::String> { + pub(super) fn sanity_check() -> Result<(), &'static str> { let mut seen_in_list = BTreeSet::new(); ensure!( Self::iter().map(|node| node.voter.id).all(|voter| seen_in_list.insert(voter)), - String::from("duplicate identified"), + "duplicate identified", ); let iter_count = Self::iter().collect::>().len() as u32; let stored_count = crate::CounterForVoters::::get(); - ensure!( - iter_count == stored_count, - format!("iter_count ({}) != voter_count ({})", iter_count, stored_count), - ); + ensure!(iter_count == stored_count, "iter_count != voter_count"); let validators = crate::CounterForValidators::::get(); let nominators = crate::CounterForNominators::::get(); - ensure!( - validators + nominators == stored_count, - format!( - "validators {} + nominators {} != voters {}", - validators, nominators, stored_count - ), - ); + ensure!(validators + nominators == stored_count, "validators + nominators != voters"); let _ = T::VoterBagThresholds::get() .into_iter() @@ -536,14 +527,14 @@ impl Bag { /// * Ensures head has no prev. /// * Ensures tail has no next. /// * Ensures there are no loops, traversal from head to tail is correct. - fn sanity_check(&self) -> Result<(), std::string::String> { + fn sanity_check(&self) -> Result<(), &'static str> { ensure!( self.head() .map(|head| head.prev().is_none()) // if there is no head, then there must not be a tail, meaning that the bag is // empty. .unwrap_or_else(|| self.tail.is_none()), - String::from("head has a prev") + "head has a prev" ); ensure!( @@ -552,7 +543,7 @@ impl Bag { // if there is no tail, then there must not be a head, meaning that the bag is // empty. .unwrap_or_else(|| self.head.is_none()), - String::from("tail has a next") + "tail has a next" ); let mut seen_in_bag = BTreeSet::new(); @@ -561,7 +552,7 @@ impl Bag { .map(|node| node.voter.id) // each voter is only seen once, thus there is no cycle within a bag .all(|voter| seen_in_bag.insert(voter)), - String::from("Duplicate found in bag.") + "Duplicate found in bag" ); Ok(()) @@ -936,7 +927,6 @@ pub mod make_bags { mod voter_list { use super::*; use crate::mock::*; - use frame_support::traits::Currency; #[test] fn basic_setup_works() { From 2f83f164f907a5479845edd7bf1ed168475a4a74 Mon Sep 17 00:00:00 2001 From: kianenigma Date: Thu, 22 Jul 2021 12:23:17 +0200 Subject: [PATCH 75/82] add logs --- frame/staking/src/voter_bags.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 687f24f539178..d5bae70b29f46 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -172,7 +172,7 @@ impl VoterList { for voter in voters.into_iter() { let weight = weight_of(&voter.id); let bag = notional_bag_for::(weight); - crate::log!(debug, "inserting {:?} into bag {:?}", voter, bag); + crate::log!(debug, "inserting {:?} with weight {} into bag {:?}", voter, weight, bag); bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); count += 1; } From 0df47b0bfa8a1e05a45ffcb1613e0bf0c1ca839a Mon Sep 17 00:00:00 2001 From: Kian Paimani <5588131+kianenigma@users.noreply.github.com> Date: Fri, 23 Jul 2021 02:09:47 +0200 Subject: [PATCH 76/82] Glue the new staking bags to the election snapshot (#9415) * Glue the new staking bags to the election snapshot * add CheckedRem (#9412) * add CheckedRem * fix * Run fmt * Test comment Co-authored-by: Xiliang Chen Co-authored-by: emostov <32168567+emostov@users.noreply.github.com> --- bin/node/runtime/voter-bags/src/main.rs | 6 +- .../election-provider-multi-phase/src/lib.rs | 61 ++++++++++++++++++- .../election-provider-multi-phase/src/mock.rs | 8 ++- frame/staking/src/lib.rs | 18 +++--- frame/staking/src/tests.rs | 30 +++++++-- frame/staking/src/voter_bags.rs | 13 ++-- primitives/arithmetic/src/traits.rs | 6 +- primitives/npos-elections/src/lib.rs | 6 +- 8 files changed, 114 insertions(+), 34 deletions(-) diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs index 1340285c29a1a..a92af37fb5bf8 100644 --- a/bin/node/runtime/voter-bags/src/main.rs +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -17,7 +17,7 @@ //! Make the set of voting bag thresholds to be used in `voter_bags.rs`. -use pallet_staking::{voter_bags::make_bags::generate_thresholds_module}; +use pallet_staking::voter_bags::make_bags::generate_thresholds_module; use std::path::PathBuf; use structopt::StructOpt; @@ -34,7 +34,5 @@ struct Opt { fn main() -> Result<(), std::io::Error> { let Opt { n_bags, output } = Opt::from_args(); let mut ext = sp_io::TestExternalities::new_empty(); - ext.execute_with(|| { - generate_thresholds_module::(n_bags, &output) - }) + ext.execute_with(|| generate_thresholds_module::(n_bags, &output)) } diff --git a/frame/election-provider-multi-phase/src/lib.rs b/frame/election-provider-multi-phase/src/lib.rs index 905492d6ca04c..3841817d04f8b 100644 --- a/frame/election-provider-multi-phase/src/lib.rs +++ b/frame/election-provider-multi-phase/src/lib.rs @@ -632,6 +632,15 @@ pub mod pallet { #[pallet::constant] type SignedDepositWeight: Get>; + /// The number of snapshot voters to fetch per block. + /// + /// In the future, this value will make more sense with multi-block snapshot. + /// + /// Also, note the data type: If the voters are represented by a `u32` in `type + /// CompactSolution`, the same `u32` is used here to ensure bounds are respected. + #[pallet::constant] + type VoterSnapshotPerBlock: Get>; + /// Handler for the slashed deposits. type SlashHandler: OnUnbalanced>; @@ -1285,8 +1294,11 @@ impl Pallet { /// /// Returns `Ok(consumed_weight)` if operation is okay. pub fn create_snapshot() -> Result { + // we don't impose any limits on the targets for now, the assumption is that + // `T::DataProvider` will sensibly return small values to use. let target_limit = >::max_value().saturated_into::(); - let voter_limit = >::max_value().saturated_into::(); + // for now we have just a single block snapshot. + let voter_limit = T::VoterSnapshotPerBlock::get().saturated_into::(); let (targets, w1) = T::DataProvider::targets(Some(target_limit)).map_err(ElectionError::DataProvider)?; @@ -1970,8 +1982,8 @@ mod tests { }) } - #[test] - fn snapshot_creation_fails_if_too_big() { + fn snapshot_too_big_failure_onchain_fallback() { + // the `MockStaking` is designed such that if it has too many targets, it simply fails. ExtBuilder::default().build_and_execute(|| { Targets::set((0..(TargetIndex::max_value() as AccountId) + 1).collect::>()); @@ -1987,6 +1999,49 @@ mod tests { roll_to(29); let (supports, _) = MultiPhase::elect().unwrap(); assert!(supports.len() > 0); + }); + } + + #[test] + fn snapshot_too_big_failure_no_fallback() { + // and if the backup mode is nothing, we go into the emergency mode.. + ExtBuilder::default().fallback(FallbackStrategy::Nothing).build_and_execute(|| { + crate::mock::Targets::set( + (0..(TargetIndex::max_value() as AccountId) + 1).collect::>(), + ); + + // Signed phase failed to open. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + // Unsigned phase failed to open. + roll_to(25); + assert_eq!(MultiPhase::current_phase(), Phase::Off); + + roll_to(29); + let err = MultiPhase::elect().unwrap_err(); + assert_eq!(err, ElectionError::NoFallbackConfigured); + assert_eq!(MultiPhase::current_phase(), Phase::Emergency); + }); + } + + #[test] + fn snapshot_too_big_truncate() { + // but if there are too many voters, we simply truncate them. + ExtBuilder::default().build_and_execute(|| { + // we have 8 voters in total. + assert_eq!(crate::mock::Voters::get().len(), 8); + // but we want to take 4. + crate::mock::VoterSnapshotPerBlock::set(2); + + // Signed phase opens just fine. + roll_to(15); + assert_eq!(MultiPhase::current_phase(), Phase::Signed); + + assert_eq!( + MultiPhase::snapshot_metadata().unwrap(), + SolutionOrSnapshotSize { voters: 2, targets: 4 } + ); }) } diff --git a/frame/election-provider-multi-phase/src/mock.rs b/frame/election-provider-multi-phase/src/mock.rs index c5007733c1e33..8e6424844f1e4 100644 --- a/frame/election-provider-multi-phase/src/mock.rs +++ b/frame/election-provider-multi-phase/src/mock.rs @@ -270,6 +270,7 @@ parameter_types! { pub static MinerMaxWeight: Weight = BlockWeights::get().max_block; pub static MinerMaxLength: u32 = 256; pub static MockWeightInfo: bool = false; + pub static VoterSnapshotPerBlock: VoterIndex = u32::max_value(); pub static EpochLength: u64 = 30; } @@ -379,6 +380,7 @@ impl crate::Config for Runtime { type Fallback = Fallback; type ForceOrigin = frame_system::EnsureRoot; type CompactSolution = TestCompact; + type VoterSnapshotPerBlock = VoterSnapshotPerBlock; } impl frame_system::offchain::SendTransactionTypes for Runtime @@ -410,9 +412,9 @@ impl ElectionDataProvider for StakingMock { fn voters( maybe_max_len: Option, ) -> data_provider::Result<(Vec<(AccountId, VoteWeight, Vec)>, Weight)> { - let voters = Voters::get(); - if maybe_max_len.map_or(false, |max_len| voters.len() > max_len) { - return Err("Voters too big") + let mut voters = Voters::get(); + if let Some(max_len) = maybe_max_len { + voters.truncate(max_len) } Ok((voters, 0)) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 4ac2b7404d0f3..3574c34602bbb 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -273,10 +273,10 @@ #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(any(feature = "runtime-benchmarks", test))] -pub mod testing_utils; #[cfg(any(feature = "runtime-benchmarks", test))] pub mod benchmarking; +#[cfg(any(feature = "runtime-benchmarks", test))] +pub mod testing_utils; #[cfg(test)] pub(crate) mod mock; @@ -3111,12 +3111,6 @@ impl Pallet { maybe_max_len: Option, voter_count: usize, ) -> Vec> { - debug_assert_eq!( - voter_count, - VoterList::::decode_len().unwrap_or_default(), - "voter_count must be accurate", - ); - let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); let weight_of = Self::weight_of_fn(); @@ -3240,8 +3234,15 @@ impl let nominator_count = CounterForNominators::::get(); let validator_count = CounterForValidators::::get(); let voter_count = nominator_count.saturating_add(validator_count) as usize; + + // check a few counters one last time... debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); + debug_assert_eq!( + voter_count, + VoterList::::decode_len().unwrap_or_default(), + "voter_count must be accurate", + ); let slashing_span_count = >::iter().count(); let weight = T::WeightInfo::get_npos_voters( @@ -3249,6 +3250,7 @@ impl validator_count, slashing_span_count as u32, ); + Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index cec490cdbfb67..4ae38611492e5 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -4006,9 +4006,21 @@ mod election_data_provider { } #[test] - fn respects_len_limits() { - ExtBuilder::default().build_and_execute(|| { + fn respects_snapshot_len_limits() { + ExtBuilder::default().validator_pool(true).build_and_execute(|| { + // sum of all validators and nominators who'd be voters. + assert_eq!(VoterList::::decode_len().unwrap(), 5); + + // if limits is less.. assert_eq!(Staking::voters(Some(1)).unwrap().0.len(), 1); + + // if limit is equal.. + assert_eq!(Staking::voters(Some(5)).unwrap().0.len(), 5); + + // if limit is more. + assert_eq!(Staking::voters(Some(55)).unwrap().0.len(), 5); + + // if target limit is less, then we return an error. assert_eq!(Staking::targets(Some(1)).unwrap_err(), "Target snapshot too big"); }); } @@ -4068,12 +4080,22 @@ mod election_data_provider { #[test] #[should_panic] - fn count_check_works() { + fn count_check_prevents_validator_insert() { ExtBuilder::default().build_and_execute(|| { // We should never insert into the validators or nominators map directly as this will // not keep track of the count. This test should panic as we verify the count is accurate - // after every test using the `post_checks` in `mock`. + // after every test using the `post_conditions` in `mock`. Validators::::insert(987654321, ValidatorPrefs::default()); + }) + } + + #[test] + #[should_panic] + fn count_check_prevents_nominator_insert() { + ExtBuilder::default().build_and_execute(|| { + // We should never insert into the validators or nominators map directly as this will + // not keep track of the count. This test should panic as we verify the count is accurate + // after every test using the `post_conditions` in `mock`. Nominators::::insert( 987654321, Nominations { diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index d5bae70b29f46..1927c4c6c049c 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -620,15 +620,13 @@ impl Node { weight_of: impl Fn(&T::AccountId) -> VoteWeight, slashing_spans: &BTreeMap, SlashingSpans>, ) -> Option> { + let voter_weight = weight_of(&self.voter.id); match self.voter.voter_type { - VoterType::Validator => Some(( - self.voter.id.clone(), - weight_of(&self.voter.id), - sp_std::vec![self.voter.id.clone()], - )), + VoterType::Validator => + Some((self.voter.id.clone(), voter_weight, sp_std::vec![self.voter.id.clone()])), VoterType::Nominator => { let Nominations { submitted_in, mut targets, .. } = - Nominators::::get(self.voter.id.clone())?; + Nominators::::get(&self.voter.id)?; // Filter out nomination targets which were nominated before the most recent // slashing span. targets.retain(|stash| { @@ -637,8 +635,7 @@ impl Node { .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) }); - (!targets.is_empty()) - .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) + (!targets.is_empty()).then(move || (self.voter.id.clone(), voter_weight, targets)) }, } } diff --git a/primitives/arithmetic/src/traits.rs b/primitives/arithmetic/src/traits.rs index a441a0dcbc08d..53341117b1fee 100644 --- a/primitives/arithmetic/src/traits.rs +++ b/primitives/arithmetic/src/traits.rs @@ -20,8 +20,8 @@ use codec::HasCompact; pub use integer_sqrt::IntegerSquareRoot; pub use num_traits::{ - checked_pow, Bounded, CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedShl, CheckedShr, - CheckedSub, One, Signed, Unsigned, Zero, + checked_pow, Bounded, CheckedAdd, CheckedDiv, CheckedMul, CheckedNeg, CheckedRem, CheckedShl, + CheckedShr, CheckedSub, One, Signed, Unsigned, Zero, }; use sp_std::{ self, @@ -55,6 +55,7 @@ pub trait BaseArithmetic: + CheckedSub + CheckedMul + CheckedDiv + + CheckedRem + Saturating + PartialOrd + Ord @@ -109,6 +110,7 @@ impl< + CheckedSub + CheckedMul + CheckedDiv + + CheckedRem + Saturating + PartialOrd + Ord diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index ad2b3229a0881..ff4919876aff1 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -155,7 +155,8 @@ where + Debug + Copy + Clone - + Bounded; + + Bounded + + Encode; /// The target type. Needs to be an index (convert to usize). type Target: UniqueSaturatedInto @@ -164,7 +165,8 @@ where + Debug + Copy + Clone - + Bounded; + + Bounded + + Encode; /// The weight/accuracy type of each vote. type Accuracy: PerThing128; From 8706404fccd542ca157c347cbaa7f69f4aa2ebbf Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Tue, 27 Jul 2021 15:30:21 -0700 Subject: [PATCH 77/82] Update node runtime with VoterSnapshotPerBlock --- bin/node/runtime/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 9c7c8a1d022d9..b319bcab92eb6 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -548,6 +548,8 @@ parameter_types! { *RuntimeBlockLength::get() .max .get(DispatchClass::Normal); + + pub const VoterSnapshotPerBlock: u32 = u32::max_value(); } sp_npos_elections::generate_solution_type!( @@ -602,6 +604,7 @@ impl pallet_election_provider_multi_phase::Config for Runtime { type WeightInfo = pallet_election_provider_multi_phase::weights::SubstrateWeight; type ForceOrigin = EnsureRootOrHalfCouncil; type BenchmarkingConfig = BenchmarkConfig; + type VoterSnapshotPerBlock = VoterSnapshotPerBlock; } parameter_types! { From ced60772e9dc3c71e014971d00a6471dc804a7d4 Mon Sep 17 00:00:00 2001 From: Zeke Mostov <32168567+emostov@users.noreply.github.com> Date: Wed, 28 Jul 2021 09:32:04 -0700 Subject: [PATCH 78/82] Unit test for pallet-staking unsorted bags feature (targets #9081) (#9422) * impl notional_bag_for_works * Add tests: insert_as_works & insert_works * Impl test: remove_works * Trivial cleaning * Add test: update_position_for_works * Write out edge case; probably can delete later * Add test: bags::get_works * Add test: remove_node_happy_path_works * Add test: remove_node_bad_paths_documented * WIP: voting_data_works * done * Improve test voting_data_works * Add comment * Fill out test basic_setup_works * Update: iteration_is_semi_sorted * Improve remove_works * Update update_position_for_works; create set_ledger_and_free_balance * Improve get_works * Improve storage clean up checks in remove test * Test: impl rebag_works + insert_and_remove_works * forgot file - Test: impl rebag_works + insert_and_remove_works * Small tweak * Update voter_bags test to reflect unused bags are removed * Unbond & Rebond: do_rebag * Prevent infinite loops with duplicate tail insert * Check iter.count on voter list in pre-migrate * undo strang fmt comment stuff * Add in todo Co-authored-by: kianenigma --- frame/staking/src/lib.rs | 4 +- frame/staking/src/mock.rs | 45 +- frame/staking/src/tests.rs | 151 +++--- frame/staking/src/voter_bags.rs | 845 +++++++++++++++++++++++++------- 4 files changed, 776 insertions(+), 269 deletions(-) diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 08d162c9fcedc..6121d3a3affa8 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -766,7 +766,7 @@ pub mod migrations { pub fn pre_migrate() -> Result<(), &'static str> { ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); ensure!( - VoterList::::decode_len().unwrap_or_default() == 0, + VoterList::::iter().count() == 0, "voter list already exists" ); Ok(()) @@ -1781,6 +1781,7 @@ pub mod pallet { let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); ledger.unlocking.push(UnlockChunk { value, era }); Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); Self::deposit_event(Event::::Unbonded(ledger.stash, value)); } Ok(()) @@ -2273,6 +2274,7 @@ pub mod pallet { Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); Ok(Some( 35 * WEIGHT_PER_MICROS + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index c16d49f7dce16..a08b2e671042d 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -493,13 +493,23 @@ impl ExtBuilder { ext.execute_with(test); ext.execute_with(post_conditions); } + /// WARNING: This should only be use for testing `VoterList` api or lower. + pub fn build_and_execute_without_check_count(self, test: impl FnOnce() -> ()) { + let mut ext = self.build(); + ext.execute_with(test); + ext.execute_with(post_conditions_without_check_count); + } } fn post_conditions() { + post_conditions_without_check_count(); + check_count(); +} + +fn post_conditions_without_check_count() { check_nominators(); check_exposures(); check_ledgers(); - check_count(); } fn check_count() { @@ -602,10 +612,14 @@ pub(crate) fn current_era() -> EraIndex { Staking::current_era().unwrap() } -pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { +pub(crate) fn bond(stash: AccountId, ctrl: AccountId, val: Balance) { let _ = Balances::make_free_balance_be(&stash, val); let _ = Balances::make_free_balance_be(&ctrl, val); assert_ok!(Staking::bond(Origin::signed(stash), ctrl, val, RewardDestination::Controller)); +} + +pub(crate) fn bond_validator(stash: AccountId, ctrl: AccountId, val: Balance) { + bond(stash, ctrl, val); assert_ok!(Staking::validate(Origin::signed(ctrl), ValidatorPrefs::default())); } @@ -615,9 +629,7 @@ pub(crate) fn bond_nominator( val: Balance, target: Vec, ) { - let _ = Balances::make_free_balance_be(&stash, val); - let _ = Balances::make_free_balance_be(&ctrl, val); - assert_ok!(Staking::bond(Origin::signed(stash), ctrl, val, RewardDestination::Controller)); + bond(stash, ctrl, val); assert_ok!(Staking::nominate(Origin::signed(ctrl), target)); } @@ -833,3 +845,26 @@ pub(crate) fn get_bags() -> Vec<(VoteWeight, Vec)> { }) .collect::>() } + +pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { + bag.iter().map(|n| n.voter().id).collect::>() +} + +pub(crate) fn get_voter_list_as_ids() -> Vec { + VoterList::::iter().map(|n| n.voter().id).collect::>() +} + +pub(crate) fn get_voter_list_as_voters() -> Vec> { + VoterList::::iter().map(|node| node.voter().clone()).collect::>() +} + +// Useful for when you want to change the effectively bonded value but you don't want to use +// the bond extrinsics because they implicitly rebag. +pub(crate) fn set_ledger_and_free_balance(account: &AccountId, value: Balance) { + Balances::make_free_balance_be(account, value); + let controller = Staking::bonded(account).unwrap(); + let mut ledger = Staking::ledger(&controller).unwrap(); + ledger.total = value; + ledger.active = value; + Staking::update_ledger(&controller, &ledger); +} diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 4ae38611492e5..11432e6d68e3d 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3827,88 +3827,87 @@ fn on_finalize_weight_is_nonzero() { // end-to-end nodes of the voter bags operation. mod voter_bags { + use super::Origin; + use crate::{mock::*, ValidatorPrefs}; + use frame_support::{assert_ok, traits::Currency}; #[test] - fn rebag_works() { - todo!() - } -} -/* -// TODO: this needs some love, retire it in favour of the one above. Use the mock data, don't make -// it complicated with data setup, use the simplest data possible, instead check multiple -// edge-cases. -#[test] -fn test_rebag() { - use crate::{ - testing_utils::create_stash_controller, - voter_bags::{Bag, Node}, - }; - use frame_system::RawOrigin; - - /// Make a validator and return its stash - fn make_validator(n: u32, balance_factor: u32) -> Result, &'static str> { - let (stash, controller) = create_stash_controller::(n, balance_factor, Default::default()).unwrap(); - - // Bond the full value of the stash - // - // By default, `create_stash_controller` only bonds 10% of the stash. However, we're going - // to want to edit one account's bonded value to match another's, so it's simpler if 100% of - // the balance is bonded. - let balance = ::Currency::free_balance(&stash); - Staking::bond_extra(RawOrigin::Signed(stash.clone()).into(), balance).unwrap(); - Staking::validate( - RawOrigin::Signed(controller.clone()).into(), - ValidatorPrefs::default(), - ).unwrap(); - - Ok(stash) + fn insert_and_remove_works() { + // we test insert/remove indirectly via `validate`, `nominate`, and chill + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // `bond` + bond(42, 43, 2_000); + // does not insert the voter + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // `validate` + assert_ok!(Staking::validate(Origin::signed(43).into(), ValidatorPrefs::default())); + // moves the voter into a bag + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // `nominate`-ing, but not changing active stake (which implicitly calls remove) + assert_ok!(Staking::nominate(Origin::signed(43), vec![11])); + // does not change the voters position + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // `chill` + assert_ok!(Staking::chill(Origin::signed(43))); + // removes the voter + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + }); } - ExtBuilder::default().build_and_execute(|| { - // We want to have two validators: one, `stash`, is the one we will rebag. - // The other, `other_stash`, exists only so that the destination bag is not empty. - let stash = make_validator(0, 2000).unwrap(); - let other_stash = make_validator(1, 9000).unwrap(); - - // verify preconditions - let weight_of = Staking::weight_of_fn(); - let node = Node::::from_id(&stash).unwrap(); - assert_eq!( - { - let origin_bag = Bag::::get(node.bag_upper).unwrap(); - origin_bag.iter().count() - }, - 1, - "stash should be the only node in origin bag", - ); - let other_node = Node::::from_id(&other_stash).unwrap(); - assert!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); - assert_ne!( - { - let destination_bag = Bag::::get(other_node.bag_upper); - destination_bag.iter().count() - }, - 0, - "destination bag should not be empty", - ); + #[test] + fn rebag_works() { + ExtBuilder::default().build_and_execute(|| { + // add a nominator to genesis state + bond_nominator(42, 43, 20, vec![11]); + Balances::make_free_balance_be(&42, 2_000); - // Update `stash`'s value to match `other_stash`, and bond extra to update its weight. - // - // This implicitly calls rebag, so the user stays in the best bag they qualify for. - let new_balance = ::Currency::free_balance(&other_stash); - ::Currency::make_free_balance_be(&stash, new_balance); - Staking::bond_extra( - RawOrigin::Signed(stash.clone()).into(), - new_balance, - ).unwrap(); - - // node should no longer be misplaced - // note that we refresh the node, in case the storage value has changed - let node = Node::::from_id(&stash).unwrap(); - assert!(!node.is_misplaced(&weight_of), "node must be in proper place after rebag"); - }); + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (20, vec![42]), (1000, vec![11, 21, 101])]); + + // increase stake and implicitly rebag with `bond_extra` to the level of non-existent bag + assert_ok!(Staking::bond_extra(Origin::signed(42), 1_980)); // 20 + 1_980 = 2_000 + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // decrease stake within the range of the current bag + assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 + // does not change bags + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] + ); + + // reduce stake to the level of a non-existent bag + assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 + // creates the bag and moves the voter into it + assert_eq!( + get_bags(), + vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] + ); + + // increase stake by `rebond`-ing to the level of a pre-existing bag + assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 + // moves the voter to that bag + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); + + // TODO test rebag directly + }); + } } -*/ mod election_data_provider { use super::*; diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 1927c4c6c049c..04d1943164bfb 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -400,6 +400,7 @@ impl VoterList { /// appearing within the voter set. #[derive(DefaultNoBound, Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq))] pub struct Bag { head: Option>, tail: Option>, @@ -430,9 +431,18 @@ impl Bag { Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) } + /// `True` if self is empty. + pub fn is_empty(&self) -> bool { + self.head.is_none() && self.tail.is_none() + } + /// Put the bag back into storage. pub fn put(self) { - crate::VoterBags::::insert(self.bag_upper, self); + if self.is_empty() { + crate::VoterBags::::remove(self.bag_upper); + } else { + crate::VoterBags::::insert(self.bag_upper, self); + } } /// Get the head node in this bag. @@ -469,6 +479,15 @@ impl Bag { /// Storage note: this modifies storage, but only for the node. You still need to call /// `self.put()` after use. fn insert_node(&mut self, mut node: Node) { + if let Some(tail) = &self.tail { + if *tail == node.voter.id { + // this should never happen, but this check prevents a worst case infinite loop + debug_assert!(false, "system logic error: inserting a node who has the id of tail"); + crate::log!(warn, "system logic error: inserting a node who has the id of tail"); + return + }; + } + let id = node.voter.id.clone(); node.prev = self.tail.clone(); @@ -562,6 +581,7 @@ impl Bag { /// A Node is the fundamental element comprising the doubly-linked lists which for each bag. #[derive(Encode, Decode)] #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound))] +#[cfg_attr(test, derive(PartialEq, Clone))] pub struct Node { voter: Voter>, prev: Option>, @@ -670,8 +690,8 @@ impl Node { notional_bag_for::(current_weight) } - #[cfg(any(test, feature = "runtime-benchmarks"))] /// Get the underlying voter. + #[cfg(any(test, feature = "runtime-benchmarks"))] pub fn voter(&self) -> &Voter { &self.voter } @@ -924,28 +944,66 @@ pub mod make_bags { mod voter_list { use super::*; use crate::mock::*; + use frame_support::{assert_ok, assert_storage_noop, traits::Currency}; #[test] fn basic_setup_works() { + use crate::{ + CounterForNominators, CounterForValidators, CounterForVoters, VoterBags, VoterNodes, + }; + let node = |voter, prev, next| Node:: { voter, prev, next, bag_upper: 0 }; + // make sure ALL relevant data structures are setup correctly. - // TODO: we are not checking all of them yet. ExtBuilder::default().build_and_execute(|| { - assert_eq!(crate::CounterForVoters::::get(), 4); + assert_eq!(CounterForVoters::::get(), 4); + assert_eq!(VoterBagFor::::iter().count(), 4); + assert_eq!(VoterNodes::::iter().count(), 4); + assert_eq!(VoterBags::::iter().count(), 2); + assert_eq!(CounterForValidators::::get(), 3); + assert_eq!(CounterForNominators::::get(), 1); + + assert_eq!( + VoterBags::::get(10).unwrap(), + Bag:: { head: Some(31), tail: Some(31), bag_upper: 0 } + ); + assert_eq!( + VoterBags::::get(1_000).unwrap(), + Bag:: { head: Some(11), tail: Some(101), bag_upper: 0 } + ); let weight_of = Staking::weight_of_fn(); + assert_eq!(weight_of(&11), 1000); assert_eq!(VoterBagFor::::get(11).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(11).unwrap(), + node(Voter::validator(11), None, Some(21)) + ); assert_eq!(weight_of(&21), 1000); assert_eq!(VoterBagFor::::get(21).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(21).unwrap(), + node(Voter::validator(21), Some(11), Some(101)) + ); assert_eq!(weight_of(&31), 1); assert_eq!(VoterBagFor::::get(31).unwrap(), 10); + assert_eq!( + VoterNodes::::get(31).unwrap(), + node(Voter::validator(31), None, None) + ); + assert_eq!(weight_of(&41), 1000); assert_eq!(VoterBagFor::::get(41), None); // this staker is chilled! + assert_eq!(VoterNodes::::get(41), None); assert_eq!(weight_of(&101), 500); assert_eq!(VoterBagFor::::get(101).unwrap(), 1000); + assert_eq!( + VoterNodes::::get(101).unwrap(), + node(Voter::nominator(101), Some(21), None) + ); // iteration of the bags would yield: assert_eq!( @@ -954,13 +1012,48 @@ mod voter_list { // ^^ note the order of insertion in genesis! ); - assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); }) } #[test] fn notional_bag_for_works() { - todo!(); + // under a threshold gives the next threshold. + assert_eq!(notional_bag_for::(0), 10); + assert_eq!(notional_bag_for::(9), 10); + assert_eq!(notional_bag_for::(11), 20); + + // at a threshold gives that threshold. + assert_eq!(notional_bag_for::(10), 10); + + let max_explicit_threshold = *::VoterBagThresholds::get().last().unwrap(); + assert_eq!(max_explicit_threshold, 10_000); + // if the max explicit threshold is less than VoteWeight::MAX, + assert!(VoteWeight::MAX > max_explicit_threshold); + // anything above it will belong to the VoteWeight::MAX bag. + assert_eq!(notional_bag_for::(max_explicit_threshold + 1), VoteWeight::MAX); + } + + #[test] + fn remove_last_voter_in_bags_cleans_bag() { + ExtBuilder::default().build_and_execute(|| { + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])]); + + // give 31 more stake to bump it to a new bag. + Balances::make_free_balance_be(&31, 10000); + assert_ok!(Staking::bond_extra(Origin::signed(31), 10000 - 10)); + + // then the bag with bound 10 is wiped from storage. + assert_eq!(get_bags(), vec![(1000, vec![11, 21, 101]), (10_000, vec![31])]); + + // and can be recreated again as needed + bond_validator(77, 777, 10); + assert_eq!( + get_bags(), + vec![(10, vec![77]), (1000, vec![11, 21, 101]), (10_000, vec![31])] + ); + }); } #[test] @@ -976,18 +1069,29 @@ mod voter_list { vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![51, 61])], ); - // when - let iteration = VoterList::::iter().map(|node| node.voter.id).collect::>(); - // then assert_eq!( - iteration, + get_voter_list_as_ids(), vec![ 51, 61, // best bag 11, 21, 101, // middle bag 31, // last bag. ] ); + + // when adding a voter that has a higher weight than pre-existing voters in the bag + bond_validator(71, 70, 10); + + // then + assert_eq!( + get_voter_list_as_ids(), + vec![ + 51, 61, // best bag + 11, 21, 101, // middle bag + 31, + 71, // last bag; the new voter is last, because it is order of insertion + ] + ); }) } @@ -1020,239 +1124,606 @@ mod voter_list { }) } - #[test] - fn storage_is_cleaned_up_as_voters_are_removed() {} - #[test] fn insert_works() { - todo!() + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when inserting into an existing bag + bond(42, 43, 1_000); + VoterList::::insert(Voter::<_>::nominator(42), Pallet::::weight_of_fn()); + + // then + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 42, 31]); + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42])]); + + // when inserting into a non-existent bag + bond(422, 433, 1_001); + VoterList::::insert(Voter::<_>::nominator(422), Pallet::::weight_of_fn()); + + // then + assert_eq!(get_voter_list_as_ids(), vec![422, 11, 21, 101, 42, 31]); + assert_eq!( + get_bags(), + vec![(10, vec![31]), (1_000, vec![11, 21, 101, 42]), (2_000, vec![422])] + ); + }); } #[test] fn insert_as_works() { - // insert a new one with role - // update the status of already existing one. - todo!() + ExtBuilder::default().build_and_execute_without_check_count(|| { + // given + let actual = get_voter_list_as_voters(); + let mut expected: Vec> = vec![ + Voter::<_>::validator(11), + Voter::<_>::validator(21), + Voter::<_>::nominator(101), + Voter::<_>::validator(31), + ]; + assert_eq!(actual, expected); + + // when inserting a new voter + VoterList::::insert_as(&42, VoterType::Nominator); + + // then + let actual = get_voter_list_as_voters(); + expected.push(Voter::<_>::nominator(42)); + assert_eq!(actual, expected); + + // when updating the voter type of an already existing voter + VoterList::::insert_as(&42, VoterType::Validator); + + // then + let actual = get_voter_list_as_voters(); + expected[4] = Voter::<_>::validator(42); + assert_eq!(actual, expected); + }); } #[test] fn remove_works() { - todo!() + use crate::{CounterForVoters, VoterBags, VoterNodes}; + + let check_storage = |id, counter, voters, bags| { + assert!(!VoterBagFor::::contains_key(id)); + assert!(!VoterNodes::::contains_key(id)); + assert_eq!(CounterForVoters::::get(), counter); + assert_eq!(VoterBagFor::::iter().count() as u32, counter); + assert_eq!(VoterNodes::::iter().count() as u32, counter); + assert_eq!(get_voter_list_as_ids(), voters); + assert_eq!(get_bags(), bags); + }; + + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when removing a non-existent voter + VoterList::::remove(&42); + assert!(!VoterBagFor::::contains_key(42)); + assert!(!VoterNodes::::contains_key(42)); + + // then nothing changes + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(CounterForVoters::::get(), 4); + + // when removing a node from a bag with multiple nodes + VoterList::::remove(&11); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101, 31]); + check_storage( + 11, + 3, + vec![21, 101, 31], // voter list + vec![(10, vec![31]), (1_000, vec![21, 101])], // bags + ); + + // when removing a node from a bag with only one node: + VoterList::::remove(&31); + + // then + assert_eq!(get_voter_list_as_ids(), vec![21, 101]); + check_storage( + 31, + 2, + vec![21, 101], // voter list + vec![(1_000, vec![21, 101])], // bags + ); + assert!(!VoterBags::::contains_key(10)); // bag 10 is removed + + // remove remaining voters to make sure storage cleans up as expected + VoterList::::remove(&21); + check_storage( + 21, + 1, + vec![101], // voter list + vec![(1_000, vec![101])], // bags + ); + + VoterList::::remove(&101); + check_storage( + 101, + 0, + Vec::::new(), // voter list + vec![], // bags + ); + assert!(!VoterBags::::contains_key(1_000)); // bag 1_000 is removed + + // bags are deleted via removals + assert_eq!(VoterBags::::iter().count(), 0); + // nominator and validator counters are not updated at this level of the api + assert_eq!(crate::CounterForValidators::::get(), 3); + assert_eq!(crate::CounterForNominators::::get(), 1); + }); } #[test] fn update_position_for_works() { - // alter the genesis state to require a re-bag, then ensure this fixes it. Might be similar - // `rebag_works()` - todo!(); + ExtBuilder::default().build_and_execute_without_check_count(|| { + let weight_of = Staking::weight_of_fn(); + + // given a correctly placed account 31 + let node_31 = Node::::from_id(&31).unwrap(); + assert!(!node_31.is_misplaced(&weight_of)); + + // when account 31 bonds extra and needs to be moved to a non-existing higher bag + // (we can't call bond_extra, because that implicitly calls update_position_for) + set_ledger_and_free_balance(&31, 11); + + assert!(node_31.is_misplaced(&weight_of)); + assert_eq!(weight_of(&31), 11); + + // then updating position moves it to the correct bag + assert_eq!(VoterList::::update_position_for(node_31, &weight_of), Some((10, 20))); + assert_eq!(get_bags(), vec![(20, vec![31]), (1_000, vec![11, 21, 101])]); + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + + // and if you try and update the position with no change in active stake nothing changes + let node_31 = Node::::from_id(&31).unwrap(); + assert_storage_noop!(assert_eq!( + VoterList::::update_position_for(node_31, &weight_of), + None, + )); + + // when account 31 bonds extra and needs to be moved to an existing higher bag + set_ledger_and_free_balance(&31, 61); + + // then updating positions moves it to the correct bag + let node_31 = Node::::from_id(&31).unwrap(); + assert_eq!( + VoterList::::update_position_for(node_31, &weight_of), + Some((20, 1_000)) + ); + assert_eq!(get_bags(), vec![(1_000, vec![11, 21, 101, 31])]); + assert_eq!(get_voter_list_as_ids(), vec![11, 21, 101, 31]); + + // when account 31 bonds extra but should not change bags + set_ledger_and_free_balance(&31, 1_000); + + // then nothing changes + let node_31 = Node::::from_id(&31).unwrap(); + assert_storage_noop!(assert_eq!( + VoterList::::update_position_for(node_31, &weight_of), + None, + )); + }); } } #[cfg(test)] mod bags { + use super::*; + use crate::mock::*; + use frame_support::{assert_ok, assert_storage_noop}; + #[test] fn get_works() { - todo!() + use crate::VoterBags; + ExtBuilder::default().build_and_execute_without_check_count(|| { + let check_bag = |bag_upper, head, tail, ids| { + assert_storage_noop!(Bag::::get(bag_upper)); + + let bag = Bag::::get(bag_upper).unwrap(); + let bag_ids = bag.iter().map(|n| n.voter().id).collect::>(); + + assert_eq!(bag, Bag:: { head, tail, bag_upper }); + assert_eq!(bag_ids, ids); + }; + + // given uppers of bags that exist. + let existing_bag_uppers = vec![10, 1_000]; + + // we can fetch them + check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); + // (getting the same bag twice has the same results) + check_bag(existing_bag_uppers[0], Some(31), Some(31), vec![31]); + check_bag(existing_bag_uppers[1], Some(11), Some(101), vec![11, 21, 101]); + + // and all other uppers don't get bags. + ::VoterBagThresholds::get() + .iter() + .chain(iter::once(&VoteWeight::MAX)) + .filter(|bag_upper| !existing_bag_uppers.contains(bag_upper)) + .for_each(|bag_upper| { + assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); + assert!(!VoterBags::::contains_key(*bag_upper)); + }); + + // when we make a pre-existing bag empty + VoterList::::remove(&31); + + // then + assert_eq!(Bag::::get(existing_bag_uppers[0]), None) + }); } #[test] - fn insert_works() { - todo!() + #[should_panic] + fn get_panics_with_a_bad_threshold() { + // NOTE: panic is only expected with debug compilation + ExtBuilder::default().build_and_execute_without_check_count(|| { + Bag::::get(11); + }); } #[test] - fn remove_works() { - todo!() + fn insert_node_happy_paths_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let node = |voter, bag_upper| Node:: { voter, prev: None, next: None, bag_upper }; + + // when inserting into a bag with 1 node + let mut bag_10 = Bag::::get(10).unwrap(); + // (note: bags api does not care about balance or ledger) + bag_10.insert_node(node(Voter::nominator(42), bag_10.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_10), vec![31, 42]); + + // when inserting into a bag with 3 nodes + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.insert_node(node(Voter::nominator(52), bag_1000.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 52]); + + // when inserting into a new bag + let mut bag_20 = Bag::::get_or_make(20); + bag_20.insert_node(node(Voter::nominator(71), bag_20.bag_upper)); + // then + assert_eq!(bag_as_ids(&bag_20), vec![71]); + + // when inserting a node pointing to the accounts not in the bag + let voter_61 = Voter::validator(61); + let node_61 = Node:: { + voter: voter_61.clone(), + prev: Some(21), + next: Some(101), + bag_upper: 20, + }; + bag_20.insert_node(node_61); + // then ids are in order + assert_eq!(bag_as_ids(&bag_20), vec![71, 61]); + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(20, &61).unwrap(), + Node:: { voter: voter_61, prev: Some(71), next: None, bag_upper: 20 } + ); + + // state of all bags is as expected + bag_20.put(); // need to put this bag so its in the storage map + assert_eq!( + get_bags(), + vec![(10, vec![31, 42]), (20, vec![71, 61]), (1_000, vec![11, 21, 101, 52])] + ); + }); } -} -#[cfg(test)] -mod voter_node { + // Document improper ways `insert_node` may be getting used. #[test] - fn get_voting_data_works() { - todo!() + fn insert_node_bad_paths_documented() { + let node = |voter, prev, next, bag_upper| Node:: { voter, prev, next, bag_upper }; + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when inserting a node with both prev & next pointing at an account in the bag + // and an incorrect bag_upper + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let voter_42 = Voter::nominator(42); + bag_1000.insert_node(node(voter_42.clone(), Some(11), Some(11), 0)); + + // then the ids are in the correct order + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 42]); + // and when the node is re-fetched all the info is correct + assert_eq!( + Node::::get(1_000, &42).unwrap(), + node(voter_42, Some(101), None, bag_1000.bag_upper) + ); + + // given 21 is a validator in bag_1000 (and not a tail node) + let bag_1000_voter = + bag_1000.iter().map(|node| node.voter().clone()).collect::>(); + assert_eq!(bag_1000_voter[1], Voter::validator(21)); + + // when inserting a node with duplicate id 21 but as a nominator + let voter_21_nom = Voter::nominator(21); + bag_1000.insert_node(node(voter_21_nom.clone(), None, None, bag_1000.bag_upper)); + + // then all the nodes after the duplicate are lost (because it is set as the tail) + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21]); + // and the re-fetched node is a nominator with an **incorrect** prev pointer. + assert_eq!( + Node::::get(1_000, &21).unwrap(), + node(voter_21_nom, Some(42), None, bag_1000.bag_upper) + ); + }); + + ExtBuilder::default().build_and_execute_without_check_count(|| { + // when inserting a duplicate id of the head + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let voter_11 = Voter::validator(11); + bag_1000.insert_node(node(voter_11.clone(), None, None, 0)); + // then all nodes after the head are lost + assert_eq!(bag_as_ids(&bag_1000), vec![11]); + // and the re-fetched node + assert_eq!( + Node::::get(1_000, &11).unwrap(), + node(voter_11, Some(101), None, bag_1000.bag_upper) + ); + + assert_eq!( + bag_1000, + Bag { + head: Some(11), tail: Some(11), bag_upper: 1_000 + } + ) + }); } #[test] - fn is_misplaced_works() { - todo!() + #[should_panic = "system logic error: inserting a node who has the id of tail"] + fn insert_node_duplicate_tail_panics_with_debug_assert() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let node = |voter, prev, next, bag_upper| Node:: { voter, prev, next, bag_upper }; + + // given + assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101])],); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + + // when inserting a duplicate id that is already the tail + assert_eq!(bag_1000.tail, Some(101)); + let voter_101 = Voter::validator(101); + bag_1000.insert_node(node(voter_101, None, None, bag_1000.bag_upper)); // panics + }); + } + + #[test] + fn remove_node_happy_paths_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + // add some validators to genesis state + bond_validator(51, 50, 1_000); + bond_validator(61, 60, 1_000); + bond_validator(71, 70, 10); + bond_validator(81, 80, 10); + bond_validator(91, 90, 2_000); + bond_validator(161, 160, 2_000); + bond_validator(171, 170, 2_000); + bond_validator(181, 180, 2_000); + bond_validator(191, 190, 2_000); + + let mut bag_10 = Bag::::get(10).unwrap(); + let mut bag_1000 = Bag::::get(1_000).unwrap(); + let mut bag_2000 = Bag::::get(2_000).unwrap(); + + // given + assert_eq!(bag_as_ids(&bag_10), vec![31, 71, 81]); + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 101, 51, 61]); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 161, 171, 181, 191]); + + // remove node that is not pointing at head or tail + let node_101 = Node::::get(bag_1000.bag_upper, &101).unwrap(); + let node_101_pre_remove = node_101.clone(); + bag_1000.remove_node(&node_101); + assert_eq!(bag_as_ids(&bag_1000), vec![11, 21, 51, 61]); + assert_ok!(bag_1000.sanity_check()); + // node isn't mutated when its removed + assert_eq!(node_101, node_101_pre_remove); + + // remove head when its not pointing at tail + let node_11 = Node::::get(bag_1000.bag_upper, &11).unwrap(); + bag_1000.remove_node(&node_11); + assert_eq!(bag_as_ids(&bag_1000), vec![21, 51, 61]); + assert_ok!(bag_1000.sanity_check()); + + // remove tail when its not pointing at head + let node_61 = Node::::get(bag_1000.bag_upper, &61).unwrap(); + bag_1000.remove_node(&node_61); + assert_eq!(bag_as_ids(&bag_1000), vec![21, 51]); + assert_ok!(bag_1000.sanity_check()); + + // remove tail when its pointing at head + let node_51 = Node::::get(bag_1000.bag_upper, &51).unwrap(); + bag_1000.remove_node(&node_51); + assert_eq!(bag_as_ids(&bag_1000), vec![21]); + assert_ok!(bag_1000.sanity_check()); + + // remove node that is head & tail + let node_21 = Node::::get(bag_1000.bag_upper, &21).unwrap(); + bag_1000.remove_node(&node_21); + bag_1000.put(); // put into storage so get returns the updated bag + assert_eq!(Bag::::get(1_000), None); + + // remove node that is pointing at head and tail + let node_71 = Node::::get(bag_10.bag_upper, &71).unwrap(); + bag_10.remove_node(&node_71); + assert_eq!(bag_as_ids(&bag_10), vec![31, 81]); + assert_ok!(bag_10.sanity_check()); + + // remove head when pointing at tail + let node_31 = Node::::get(bag_10.bag_upper, &31).unwrap(); + bag_10.remove_node(&node_31); + assert_eq!(bag_as_ids(&bag_10), vec![81]); + assert_ok!(bag_10.sanity_check()); + bag_10.put(); // since we updated the bag's head/tail, we need to write this storage + + // remove node that is pointing at head, but not tail + let node_161 = Node::::get(bag_2000.bag_upper, &161).unwrap(); + bag_2000.remove_node(&node_161); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 181, 191]); + assert_ok!(bag_2000.sanity_check()); + + // remove node that is pointing at tail, but not head + let node_181 = Node::::get(bag_2000.bag_upper, &181).unwrap(); + bag_2000.remove_node(&node_181); + assert_eq!(bag_as_ids(&bag_2000), vec![91, 171, 191]); + assert_ok!(bag_2000.sanity_check()); + + // state of all bags is as expected + assert_eq!(get_bags(), vec![(10, vec![81]), (2_000, vec![91, 171, 191])]); + }); + } + + #[test] + fn remove_node_bad_paths_documented() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + // removing a node that is in the bag but has the wrong upper works. + + let bad_upper_node_11 = Node:: { + voter: Voter::<_>::validator(11), + prev: None, + next: Some(21), + bag_upper: 10, // should be 1_000 + }; + let mut bag_1000 = Bag::::get(1_000).unwrap(); + bag_1000.remove_node(&bad_upper_node_11); + bag_1000.put(); + + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![21, 101])]); + let bag_1000 = Bag::::get(1_000).unwrap(); + assert_ok!(bag_1000.sanity_check()); + assert_eq!(bag_1000.head, Some(21)); + assert_eq!(bag_1000.tail, Some(101)); + }); + + ExtBuilder::default().build_and_execute_without_check_count(|| { + // removing a node that is in another bag, will mess up the + // other bag. + + let node_101 = Node::::get(1_000, &101).unwrap(); + let mut bag_10 = Bag::::get(10).unwrap(); + bag_10.remove_node(&node_101); // node_101 is in bag 1_000 + bag_10.put(); + + // the node was removed from its actual bag, bag_1000. + assert_eq!(get_bags(), vec![(10, vec![31]), (1_000, vec![11, 21])]); + + // the bag removed was called on is ok. + let bag_10 = Bag::::get(10).unwrap(); + assert_eq!(bag_10.tail, Some(31)); + assert_eq!(bag_10.head, Some(31)); + + // but the bag that the node belonged to is in an invalid state + let bag_1000 = Bag::::get(1_000).unwrap(); + // because it still has the removed node as its tail. + assert_eq!(bag_1000.tail, Some(101)); + assert_eq!(bag_1000.head, Some(11)); + assert_ok!(bag_1000.sanity_check()); + }); } } -// TODO: I've created simpler versions of these tests above. We can probably remove the ones below -// now. Peter was likely not very familiar with the staking mock and he came up with these rather -// complicated test setups. Please see my versions above, we can test the same properties, easily, -// without the need to alter the stakers so much. -/* #[cfg(test)] -mod tests { - use frame_support::traits::Currency; - +mod voter_node { use super::*; use crate::mock::*; - const GENESIS_VOTER_IDS: [u64; 5] = [11, 21, 31, 41, 101]; - - /// This tests the property that when iterating through the `VoterList`, we iterate from higher - /// bags to lower. #[test] - fn iteration_is_semi_sorted() { - use rand::seq::SliceRandom; - let mut rng = rand::thread_rng(); - - // Randomly sort the list of voters. Later we'll give each of these a stake such that it - // fits into a different bag. - let voters = { - let mut v = vec![0; GENESIS_VOTER_IDS.len()]; - v.copy_from_slice(&GENESIS_VOTER_IDS); - v.shuffle(&mut rng); - v - }; + fn voting_data_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let weight_of = Staking::weight_of_fn(); - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - // initialize the voters' deposits - let mut balance = 10; - for voter_id in voters.iter().rev() { - ::Currency::make_free_balance_be(voter_id, balance); - let controller = Staking::bonded(voter_id).unwrap(); - let mut ledger = Staking::ledger(&controller).unwrap(); - ledger.total = balance; - ledger.active = balance; - Staking::update_ledger(&controller, &ledger); - Staking::do_rebag(voter_id); - - // Increase balance to the next threshold. - balance += 10; - } + // add nominator with no targets + bond_nominator(42, 43, 1_000, vec![11]); - let have_voters: Vec<_> = VoterList::::iter().map(|node| node.voter.id).collect(); - assert_eq!(voters, have_voters); - }); - } + // given + assert_eq!( + get_voter_list_as_voters(), + vec![ + Voter::validator(11), + Voter::validator(21), + Voter::nominator(101), + Voter::nominator(42), + Voter::validator(31), + ] + ); + assert_eq!(active_era(), 0); - /// This tests that we can `take` x voters, even if that quantity ends midway through a list. - #[test] - fn take_works() { - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - // initialize the voters' deposits - let mut balance = 0; // This will be 10 on the first loop iteration because 0 % 3 == 0 - for (idx, voter_id) in GENESIS_VOTER_IDS.iter().enumerate() { - if idx % 3 == 0 { - // This increases the balance by 10, which is the amount each threshold - // increases by. Thus this will increase the balance by 1 bag. - // - // This will create 2 bags, the lower threshold bag having - // 3 voters with balance 10, and the higher threshold bag having - // 2 voters with balance 20. - balance += 10; - } + let slashing_spans = + ::SlashingSpans::iter().collect::>(); + assert_eq!(slashing_spans.keys().len(), 0); // no pre-existing slashing spans - ::Currency::make_free_balance_be(voter_id, balance); - let controller = Staking::bonded(voter_id).unwrap(); - let mut ledger = Staking::ledger(&controller).unwrap(); - ledger.total = balance; - ledger.active = balance; - Staking::update_ledger(&controller, &ledger); - Staking::do_rebag(voter_id); - } + let node_11 = Node::::get(10, &11).unwrap(); + assert_eq!( + node_11.voting_data(&weight_of, &slashing_spans).unwrap(), + (11, 1_000, vec![11]) + ); - let bag_thresh10 = Bag::::get(10) - .unwrap() - .iter() - .map(|node| node.voter.id) - .collect::>(); - assert_eq!(bag_thresh10, vec![11, 21, 31]); + // getting data for a nominators with 0 slashed targets + let node_101 = Node::::get(1_000, &101).unwrap(); + assert_eq!( + node_101.voting_data(&weight_of, &slashing_spans).unwrap(), + (101, 500, vec![11, 21]) + ); + let node_42 = Node::::get(10, &42).unwrap(); + assert_eq!( + node_42.voting_data(&weight_of, &slashing_spans).unwrap(), + (42, 1_000, vec![11]) + ); - let bag_thresh20 = Bag::::get(20) - .unwrap() - .iter() - .map(|node| node.voter.id) - .collect::>(); - assert_eq!(bag_thresh20, vec![41, 101]); + // roll ahead an era so any slashes will be after the previous nominations + start_active_era(1); - let voters: Vec<_> = VoterList::::iter() - // take 4/5 from [41, 101],[11, 21, 31], demonstrating that we can do a - // take that stops mid bag. - .take(4) - .map(|node| node.voter.id) - .collect(); + // when a validator gets a slash, + add_slash(&11); + let slashing_spans = + ::SlashingSpans::iter().collect::>(); + + assert_eq!(slashing_spans.keys().cloned().collect::>(), vec![11, 42, 101]); + // then its node no longer exists + assert_eq!( + get_voter_list_as_voters(), + vec![ + Voter::validator(21), + Voter::nominator(101), + Voter::nominator(42), + Voter::validator(31), + ] + ); + // and its nominators no longer have it as a target + let node_101 = Node::::get(10, &101).unwrap(); + assert_eq!( + node_101.voting_data(&weight_of, &slashing_spans), + Some((101, 475, vec![21])), + ); - assert_eq!(voters, vec![41, 101, 11, 21]); + let node_42 = Node::::get(10, &42).unwrap(); + assert_eq!( + node_42.voting_data(&weight_of, &slashing_spans), + None, // no voting data since its 1 target has been slashed since nominating + ); }); } #[test] - fn storage_is_cleaned_up_as_voters_are_removed() { - ExtBuilder::default().validator_pool(true).build_and_execute(|| { - // Initialize voters deposits so there are 5 bags with one voter each. - let mut balance = 10; - for voter_id in GENESIS_VOTER_IDS.iter() { - ::Currency::make_free_balance_be(voter_id, balance); - let controller = Staking::bonded(voter_id).unwrap(); - let mut ledger = Staking::ledger(&controller).unwrap(); - ledger.total = balance; - ledger.active = balance; - Staking::update_ledger(&controller, &ledger); - Staking::do_rebag(voter_id); - - // Increase balance to the next threshold. - balance += 10; - } + fn is_misplaced_works() { + ExtBuilder::default().build_and_execute_without_check_count(|| { + let weight_of = Staking::weight_of_fn(); + let node_31 = Node::::get(10, &31).unwrap(); - let voter_list_storage_items_eq = |mut v: Vec| { - v.sort(); - let mut voters: Vec<_> = - VoterList::::iter().map(|node| node.voter.id).collect(); - voters.sort(); - assert_eq!(voters, v); - - let mut nodes: Vec<_> = - ::VoterNodes::iter_keys().collect(); - nodes.sort(); - assert_eq!(nodes, v); - - let mut flat_bags: Vec<_> = ::VoterBags::iter() - // We always get the bag with the Bag getter because the bag_upper - // is only initialized in the getter. - .flat_map(|(key, _bag)| Bag::::get(key).unwrap().iter()) - .map(|node| node.voter.id) - .collect(); - flat_bags.sort(); - assert_eq!(flat_bags, v); - - let mut bags_for: Vec<_> = - ::VoterBagFor::iter_keys().collect(); - bags_for.sort(); - assert_eq!(bags_for, v); - }; + // a node is properly placed if its slashable balance is in range + // of the threshold of the bag its in. + assert_eq!(Staking::slashable_balance_of(&31), 1); + assert!(!node_31.is_misplaced(&weight_of)); - let genesis_voters = vec![101, 41, 31, 21, 11]; - voter_list_storage_items_eq(genesis_voters); - assert_eq!(::CounterForVoters::get(), 5); + // and will become misplaced if its slashable balance does not + // correspond to the bag it is in. + set_ledger_and_free_balance(&31, 11); - // Remove 1 voter, - VoterList::::remove(&101); - let remaining_voters = vec![41, 31, 21, 11]; - // and assert they have been cleaned up. - voter_list_storage_items_eq(remaining_voters.clone()); - assert_eq!(::CounterForVoters::get(), 4); - - // Now remove the remaining voters so we have 0 left, - remaining_voters.iter().for_each(|v| VoterList::::remove(v)); - // and assert all of them have been cleaned up. - voter_list_storage_items_eq(vec![]); - assert_eq!(::CounterForVoters::get(), 0); - - - // TODO bags do not get cleaned up from storages - // - is this ok? I assume its ok if this is not cleaned just because voters are removed - // but it should be cleaned up if we migrate thresholds - assert_eq!(::VoterBags::iter().collect::>().len(), 6); - // and the voter list has no one in it. - assert_eq!(VoterList::::iter().collect::>().len(), 0); - assert_eq!(::VoterBagFor::iter().collect::>().len(), 0); - assert_eq!(::VoterNodes::iter().collect::>().len(), 0); + assert_eq!(Staking::slashable_balance_of(&31), 11); + assert!(node_31.is_misplaced(&weight_of)); }); } } -*/ From da4814c234a5a56fe08902c002ee69d7f30996f4 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 28 Jul 2021 10:40:49 -0700 Subject: [PATCH 79/82] Try prepare for master merge --- frame/staking/src/benchmarking.rs | 14 +- frame/staking/src/lib.rs | 2904 +--------------------------- frame/staking/src/migrations.rs | 117 ++ frame/staking/src/mock.rs | 7 +- frame/staking/src/pallet/impls.rs | 1158 +++++++++++ frame/staking/src/pallet/mod.rs | 1698 ++++++++++++++++ frame/staking/src/slashing.rs | 2 +- frame/staking/src/testing_utils.rs | 5 + frame/staking/src/tests.rs | 20 +- frame/staking/src/voter_bags.rs | 7 +- 10 files changed, 3024 insertions(+), 2908 deletions(-) create mode 100644 frame/staking/src/migrations.rs create mode 100644 frame/staking/src/pallet/impls.rs create mode 100644 frame/staking/src/pallet/mod.rs diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index e817cacda6f5b..ff4a8ba986b00 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -21,11 +21,23 @@ use super::*; use crate::Pallet as Staking; use testing_utils::*; +use frame_support::{ + pallet_prelude::*, + traits::{Currency, Get, Imbalance}, +}; +use sp_runtime::{ + traits::{CheckedSub, StaticLookup, Zero}, + Perbill, Percent, +}; +use sp_staking::SessionIndex; +use sp_std::prelude::*; + +use crate::voter_bags::VoterList; pub use frame_benchmarking::{ account, benchmarks, impl_benchmark_test_suite, whitelist_account, whitelisted_caller, }; use frame_system::RawOrigin; -use sp_runtime::traits::One; +use sp_runtime::traits::{Bounded, One}; const SEED: u32 = 0; const MAX_SPANS: u32 = 100; diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 6121d3a3affa8..993fe349a1045 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -284,43 +284,33 @@ pub(crate) mod mock; mod tests; pub mod inflation; +pub mod migrations; pub mod slashing; pub mod voter_bags; pub mod weights; +mod pallet; + use codec::{Decode, Encode, HasCompact}; -use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; +use frame_election_provider_support::VoteWeight; use frame_support::{ - pallet_prelude::*, - traits::{ - Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, Imbalance, - LockIdentifier, LockableCurrency, OnUnbalanced, UnixTime, WithdrawReasons, - }, - weights::{ - constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}, - Weight, WithPostDispatchInfo, - }, + traits::{Currency, Get}, + weights::Weight, }; -use frame_system::{ensure_root, ensure_signed, offchain::SendTransactionTypes, pallet_prelude::*}; -pub use pallet::*; -use pallet_session::historical; use sp_runtime::{ curve::PiecewiseLinear, - traits::{ - AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, - StaticLookup, Zero, - }, - DispatchError, Perbill, Percent, RuntimeDebug, + traits::{AtLeast32BitUnsigned, Convert, Saturating, Zero}, + Perbill, RuntimeDebug, }; use sp_staking::{ - offence::{Offence, OffenceDetails, OffenceError, OnOffenceHandler, ReportOffence}, + offence::{Offence, OffenceError, ReportOffence}, SessionIndex, }; -use sp_std::{collections::btree_map::BTreeMap, convert::From, prelude::*, result}; -use voter_bags::{VoterList, VoterType}; +use sp_std::{collections::btree_map::BTreeMap, convert::From, prelude::*}; pub use weights::WeightInfo; -const STAKING_ID: LockIdentifier = *b"staking "; +pub use pallet::{pallet::*, *}; + pub(crate) const LOG_TARGET: &'static str = "runtime::staking"; // syntactic sugar for logging. @@ -334,8 +324,6 @@ macro_rules! log { }; } -pub const MAX_UNLOCKING_CHUNKS: usize = 32; - /// Counter for the number of eras that have passed. pub type EraIndex = u32; @@ -757,2737 +745,6 @@ impl Default for Releases { } } -pub mod migrations { - use super::*; - - pub mod v8 { - use super::*; - - pub fn pre_migrate() -> Result<(), &'static str> { - ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); - ensure!( - VoterList::::iter().count() == 0, - "voter list already exists" - ); - Ok(()) - } - - pub fn migrate() -> Weight { - log!(info, "Migrating staking to Releases::V8_0_0"); - - let migrated = VoterList::::regenerate(); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - - StorageVersion::::put(Releases::V8_0_0); - log!( - info, - "Completed staking migration to Releases::V8_0_0 with {} voters migrated", - migrated, - ); - - T::WeightInfo::regenerate( - CounterForValidators::::get(), - CounterForNominators::::get(), - ) - .saturating_add(T::DbWeight::get().reads(2)) - } - } - - pub mod v7 { - use super::*; - - pub fn pre_migrate() -> Result<(), &'static str> { - assert!( - CounterForValidators::::get().is_zero(), - "CounterForValidators already set." - ); - assert!( - CounterForNominators::::get().is_zero(), - "CounterForNominators already set." - ); - assert!(StorageVersion::::get() == Releases::V6_0_0); - Ok(()) - } - - pub fn migrate() -> Weight { - log!(info, "Migrating staking to Releases::V7_0_0"); - let validator_count = Validators::::iter().count() as u32; - let nominator_count = Nominators::::iter().count() as u32; - - CounterForValidators::::put(validator_count); - CounterForNominators::::put(nominator_count); - - StorageVersion::::put(Releases::V7_0_0); - log!(info, "Completed staking migration to Releases::V7_0_0"); - - T::DbWeight::get() - .reads_writes(validator_count.saturating_add(nominator_count).into(), 2) - } - } - - pub mod v6 { - use super::*; - use frame_support::{generate_storage_alias, traits::Get, weights::Weight}; - - // NOTE: value type doesn't matter, we just set it to () here. - generate_storage_alias!(Staking, SnapshotValidators => Value<()>); - generate_storage_alias!(Staking, SnapshotNominators => Value<()>); - generate_storage_alias!(Staking, QueuedElected => Value<()>); - generate_storage_alias!(Staking, QueuedScore => Value<()>); - generate_storage_alias!(Staking, EraElectionStatus => Value<()>); - generate_storage_alias!(Staking, IsCurrentSessionFinal => Value<()>); - - /// check to execute prior to migration. - pub fn pre_migrate() -> Result<(), &'static str> { - // these may or may not exist. - log!(info, "SnapshotValidators.exits()? {:?}", SnapshotValidators::exists()); - log!(info, "SnapshotNominators.exits()? {:?}", SnapshotNominators::exists()); - log!(info, "QueuedElected.exits()? {:?}", QueuedElected::exists()); - log!(info, "QueuedScore.exits()? {:?}", QueuedScore::exists()); - // these must exist. - assert!( - IsCurrentSessionFinal::exists(), - "IsCurrentSessionFinal storage item not found!" - ); - assert!(EraElectionStatus::exists(), "EraElectionStatus storage item not found!"); - Ok(()) - } - - /// Migrate storage to v6. - pub fn migrate() -> Weight { - log!(info, "Migrating staking to Releases::V6_0_0"); - - SnapshotValidators::kill(); - SnapshotNominators::kill(); - QueuedElected::kill(); - QueuedScore::kill(); - EraElectionStatus::kill(); - IsCurrentSessionFinal::kill(); - - StorageVersion::::put(Releases::V6_0_0); - log!(info, "Done."); - T::DbWeight::get().writes(6 + 1) - } - } -} - -#[frame_support::pallet] -pub mod pallet { - use super::*; - - #[pallet::pallet] - #[pallet::generate_store(pub(super) trait Store)] - pub struct Pallet(_); - - #[pallet::config] - pub trait Config: frame_system::Config + SendTransactionTypes> { - /// The staking balance. - type Currency: LockableCurrency; - - /// Time used for computing era duration. - /// - /// It is guaranteed to start being called from the first `on_finalize`. Thus value at genesis - /// is not used. - type UnixTime: UnixTime; - - /// Convert a balance into a number used for election calculation. This must fit into a `u64` - /// but is allowed to be sensibly lossy. The `u64` is used to communicate with the - /// [`sp_npos_elections`] crate which accepts u64 numbers and does operations in 128. - /// Consequently, the backward convert is used convert the u128s from sp-elections back to a - /// [`BalanceOf`]. - type CurrencyToVote: CurrencyToVote>; - - /// Something that provides the election functionality. - type ElectionProvider: frame_election_provider_support::ElectionProvider< - Self::AccountId, - Self::BlockNumber, - // we only accept an election provider that has staking as data provider. - DataProvider = Pallet, - >; - - /// Something that provides the election functionality at genesis. - type GenesisElectionProvider: frame_election_provider_support::ElectionProvider< - Self::AccountId, - Self::BlockNumber, - DataProvider = Pallet, - >; - - /// Maximum number of nominations per nominator. - const MAX_NOMINATIONS: u32; - - /// Tokens have been minted and are unused for validator-reward. - /// See [Era payout](./index.html#era-payout). - type RewardRemainder: OnUnbalanced>; - - /// The overarching event type. - type Event: From> + IsType<::Event>; - - /// Handler for the unbalanced reduction when slashing a staker. - type Slash: OnUnbalanced>; - - /// Handler for the unbalanced increment when rewarding a staker. - type Reward: OnUnbalanced>; - - /// Number of sessions per era. - #[pallet::constant] - type SessionsPerEra: Get; - - /// Number of eras that staked funds must remain bonded for. - #[pallet::constant] - type BondingDuration: Get; - - /// Number of eras that slashes are deferred by, after computation. - /// - /// This should be less than the bonding duration. Set to 0 if slashes - /// should be applied immediately, without opportunity for intervention. - #[pallet::constant] - type SlashDeferDuration: Get; - - /// The origin which can cancel a deferred slash. Root can always do this. - type SlashCancelOrigin: EnsureOrigin; - - /// Interface for interacting with a session pallet. - type SessionInterface: self::SessionInterface; - - /// The payout for validators and the system for the current era. - /// See [Era payout](./index.html#era-payout). - type EraPayout: EraPayout>; - - /// Something that can estimate the next session change, accurately or as a best effort guess. - type NextNewSession: EstimateNextNewSession; - - /// The maximum number of nominators rewarded for each validator. - /// - /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim - /// their reward. This used to limit the i/o cost for the nominator payout. - #[pallet::constant] - type MaxNominatorRewardedPerValidator: Get; - - /// Weight information for extrinsics in this pallet. - type WeightInfo: WeightInfo; - - /// The list of thresholds separating the various voter bags. - /// - /// Voters are separated into unsorted bags according to their vote weight. This specifies - /// the thresholds separating the bags. A voter's bag is the largest bag for which the - /// voter's weight is less than or equal to its upper threshold. - /// - /// When voters are iterated, higher bags are iterated completely before lower bags. This - /// means that iteration is _semi-sorted_: voters of higher weight tend to come before - /// voters of lower weight, but peer voters within a particular bag are sorted in insertion - /// order. - /// - /// # Expressing the constant - /// - /// This constant must be sorted in strictly increasing order. Duplicate items are not - /// permitted. - /// - /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be - /// specified within the bag. For any two threshold lists, if one ends with - /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists - /// will behave identically. - /// - /// # Calculation - /// - /// It is recommended to generate the set of thresholds in a geometric series, such that - /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * - /// constant_ratio).max(threshold[k] + 1)` for all `k`. - /// - /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use - /// them, the `make-bags` feature must be enabled. - /// - /// # Examples - /// - /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, - /// and iteration is strictly in insertion order. - /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is equal to 2. - /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined - /// according to the procedure given above, then the constant ratio is approximately equal - /// to 1.248. - /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will - /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. - /// - /// # Migration - /// - /// In the event that this list ever changes, a copy of the old bags list must be retained. - /// With that `VoterList::migrate` can be called, which will perform the appropriate - /// migration. - #[pallet::constant] - type VoterBagThresholds: Get<&'static [VoteWeight]>; - } - - #[pallet::extra_constants] - impl Pallet { - // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. - #[allow(non_snake_case)] - fn MaxNominations() -> u32 { - T::MAX_NOMINATIONS - } - } - - #[pallet::type_value] - pub(crate) fn HistoryDepthOnEmpty() -> u32 { - 84u32 - } - - /// Number of eras to keep in history. - /// - /// Information is kept for eras in `[current_era - history_depth; current_era]`. - /// - /// Must be more than the number of eras delayed by session otherwise. I.e. active era must - /// always be in history. I.e. `active_era > current_era - history_depth` must be - /// guaranteed. - #[pallet::storage] - #[pallet::getter(fn history_depth)] - pub(crate) type HistoryDepth = StorageValue<_, u32, ValueQuery, HistoryDepthOnEmpty>; - - /// The ideal number of staking participants. - #[pallet::storage] - #[pallet::getter(fn validator_count)] - pub type ValidatorCount = StorageValue<_, u32, ValueQuery>; - - /// Minimum number of staking participants before emergency conditions are imposed. - #[pallet::storage] - #[pallet::getter(fn minimum_validator_count)] - pub type MinimumValidatorCount = StorageValue<_, u32, ValueQuery>; - - /// Any validators that may never be slashed or forcibly kicked. It's a Vec since they're - /// easy to initialize and the performance hit is minimal (we expect no more than four - /// invulnerables) and restricted to testnets. - #[pallet::storage] - #[pallet::getter(fn invulnerables)] - pub type Invulnerables = StorageValue<_, Vec, ValueQuery>; - - /// Map from all locked "stash" accounts to the controller account. - #[pallet::storage] - #[pallet::getter(fn bonded)] - pub type Bonded = StorageMap<_, Twox64Concat, T::AccountId, T::AccountId>; - - /// The minimum active bond to become and maintain the role of a nominator. - #[pallet::storage] - pub type MinNominatorBond = StorageValue<_, BalanceOf, ValueQuery>; - - /// The minimum active bond to become and maintain the role of a validator. - #[pallet::storage] - pub type MinValidatorBond = StorageValue<_, BalanceOf, ValueQuery>; - - /// Map from all (unlocked) "controller" accounts to the info regarding the staking. - #[pallet::storage] - #[pallet::getter(fn ledger)] - pub type Ledger = - StorageMap<_, Blake2_128Concat, T::AccountId, StakingLedger>>; - - /// Where the reward payment should be made. Keyed by stash. - #[pallet::storage] - #[pallet::getter(fn payee)] - pub type Payee = - StorageMap<_, Twox64Concat, T::AccountId, RewardDestination, ValueQuery>; - - /// The map from (wannabe) validator stash key to the preferences of that validator. - /// - /// When updating this storage item, you must also update the `CounterForValidators`. - #[pallet::storage] - #[pallet::getter(fn validators)] - pub type Validators = - StorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; - - /// A tracker to keep count of the number of items in the `Validators` map. - #[pallet::storage] - pub type CounterForValidators = StorageValue<_, u32, ValueQuery>; - - /// The maximum validator count before we stop allowing new validators to join. - /// - /// When this value is not set, no limits are enforced. - #[pallet::storage] - pub type MaxValidatorsCount = StorageValue<_, u32, OptionQuery>; - - /// The map from nominator stash key to the set of stash keys of all validators to nominate. - /// - /// When updating this storage item, you must also update the `CounterForNominators`. - #[pallet::storage] - #[pallet::getter(fn nominators)] - pub type Nominators = - StorageMap<_, Twox64Concat, T::AccountId, Nominations>; - - /// A tracker to keep count of the number of items in the `Nominators` map. - #[pallet::storage] - pub type CounterForNominators = StorageValue<_, u32, ValueQuery>; - - /// The maximum nominator count before we stop allowing new validators to join. - /// - /// When this value is not set, no limits are enforced. - #[pallet::storage] - pub type MaxNominatorsCount = StorageValue<_, u32, OptionQuery>; - - /// The current era index. - /// - /// This is the latest planned era, depending on how the Session pallet queues the validator - /// set, it might be active or not. - #[pallet::storage] - #[pallet::getter(fn current_era)] - pub type CurrentEra = StorageValue<_, EraIndex>; - - /// The active era information, it holds index and start. - /// - /// The active era is the era being currently rewarded. Validator set of this era must be - /// equal to [`SessionInterface::validators`]. - #[pallet::storage] - #[pallet::getter(fn active_era)] - pub type ActiveEra = StorageValue<_, ActiveEraInfo>; - - /// The session index at which the era start for the last `HISTORY_DEPTH` eras. - /// - /// Note: This tracks the starting session (i.e. session index when era start being active) - /// for the eras in `[CurrentEra - HISTORY_DEPTH, CurrentEra]`. - #[pallet::storage] - #[pallet::getter(fn eras_start_session_index)] - pub type ErasStartSessionIndex = StorageMap<_, Twox64Concat, EraIndex, SessionIndex>; - - /// Exposure of validator at era. - /// - /// This is keyed first by the era index to allow bulk deletion and then the stash account. - /// - /// Is it removed after `HISTORY_DEPTH` eras. - /// If stakers hasn't been set or has been removed then empty exposure is returned. - #[pallet::storage] - #[pallet::getter(fn eras_stakers)] - pub type ErasStakers = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - Exposure>, - ValueQuery, - >; - - /// Clipped Exposure of validator at era. - /// - /// This is similar to [`ErasStakers`] but number of nominators exposed is reduced to the - /// `T::MaxNominatorRewardedPerValidator` biggest stakers. - /// (Note: the field `total` and `own` of the exposure remains unchanged). - /// This is used to limit the i/o cost for the nominator payout. - /// - /// This is keyed fist by the era index to allow bulk deletion and then the stash account. - /// - /// Is it removed after `HISTORY_DEPTH` eras. - /// If stakers hasn't been set or has been removed then empty exposure is returned. - #[pallet::storage] - #[pallet::getter(fn eras_stakers_clipped)] - pub type ErasStakersClipped = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - Exposure>, - ValueQuery, - >; - - /// Similar to `ErasStakers`, this holds the preferences of validators. - /// - /// This is keyed first by the era index to allow bulk deletion and then the stash account. - /// - /// Is it removed after `HISTORY_DEPTH` eras. - // If prefs hasn't been set or has been removed then 0 commission is returned. - #[pallet::storage] - #[pallet::getter(fn eras_validator_prefs)] - pub type ErasValidatorPrefs = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - ValidatorPrefs, - ValueQuery, - >; - - /// The total validator era payout for the last `HISTORY_DEPTH` eras. - /// - /// Eras that haven't finished yet or has been removed doesn't have reward. - #[pallet::storage] - #[pallet::getter(fn eras_validator_reward)] - pub type ErasValidatorReward = StorageMap<_, Twox64Concat, EraIndex, BalanceOf>; - - /// Rewards for the last `HISTORY_DEPTH` eras. - /// If reward hasn't been set or has been removed then 0 reward is returned. - #[pallet::storage] - #[pallet::getter(fn eras_reward_points)] - pub type ErasRewardPoints = - StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; - - /// The total amount staked for the last `HISTORY_DEPTH` eras. - /// If total hasn't been set or has been removed then 0 stake is returned. - #[pallet::storage] - #[pallet::getter(fn eras_total_stake)] - pub type ErasTotalStake = - StorageMap<_, Twox64Concat, EraIndex, BalanceOf, ValueQuery>; - - /// Mode of era forcing. - #[pallet::storage] - #[pallet::getter(fn force_era)] - pub type ForceEra = StorageValue<_, Forcing, ValueQuery>; - - /// The percentage of the slash that is distributed to reporters. - /// - /// The rest of the slashed value is handled by the `Slash`. - #[pallet::storage] - #[pallet::getter(fn slash_reward_fraction)] - pub type SlashRewardFraction = StorageValue<_, Perbill, ValueQuery>; - - /// The amount of currency given to reporters of a slash event which was - /// canceled by extraordinary circumstances (e.g. governance). - #[pallet::storage] - #[pallet::getter(fn canceled_payout)] - pub type CanceledSlashPayout = StorageValue<_, BalanceOf, ValueQuery>; - - /// All unapplied slashes that are queued for later. - #[pallet::storage] - pub type UnappliedSlashes = StorageMap< - _, - Twox64Concat, - EraIndex, - Vec>>, - ValueQuery, - >; - - /// A mapping from still-bonded eras to the first session index of that era. - /// - /// Must contains information for eras for the range: - /// `[active_era - bounding_duration; active_era]` - #[pallet::storage] - pub(crate) type BondedEras = - StorageValue<_, Vec<(EraIndex, SessionIndex)>, ValueQuery>; - - /// All slashing events on validators, mapped by era to the highest slash proportion - /// and slash value of the era. - #[pallet::storage] - pub(crate) type ValidatorSlashInEra = StorageDoubleMap< - _, - Twox64Concat, - EraIndex, - Twox64Concat, - T::AccountId, - (Perbill, BalanceOf), - >; - - /// All slashing events on nominators, mapped by era to the highest slash value of the era. - #[pallet::storage] - pub(crate) type NominatorSlashInEra = - StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, BalanceOf>; - - /// Slashing spans for stash accounts. - #[pallet::storage] - pub(crate) type SlashingSpans = - StorageMap<_, Twox64Concat, T::AccountId, slashing::SlashingSpans>; - - /// Records information about the maximum slash of a stash within a slashing span, - /// as well as how much reward has been paid out. - #[pallet::storage] - pub(crate) type SpanSlash = StorageMap< - _, - Twox64Concat, - (T::AccountId, slashing::SpanIndex), - slashing::SpanRecord>, - ValueQuery, - >; - - /// The earliest era for which we have a pending, unapplied slash. - #[pallet::storage] - pub(crate) type EarliestUnappliedSlash = StorageValue<_, EraIndex>; - - /// The last planned session scheduled by the session pallet. - /// - /// This is basically in sync with the call to [`pallet_session::SessionManager::new_session`]. - #[pallet::storage] - #[pallet::getter(fn current_planned_session)] - pub type CurrentPlannedSession = StorageValue<_, SessionIndex, ValueQuery>; - - /// True if network has been upgraded to this version. - /// Storage version of the pallet. - /// - /// This is set to v7.0.0 for new networks. - #[pallet::storage] - pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; - - // The next storage items collectively comprise the voter bags: a composite data structure - // designed to allow efficient iteration of the top N voters by stake, mostly. See - // `mod voter_bags` for details. - // - // In each of these items, voter bags are indexed by their upper weight threshold. - - /// How many voters are registered. - #[pallet::storage] - pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; - - /// Which bag currently contains a particular voter. - /// - /// This may not be the appropriate bag for the voter's weight if they have been rewarded or - /// slashed. - #[pallet::storage] - pub(crate) type VoterBagFor = - StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; - - /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which - /// mainly exists to store head and tail pointers to the appropriate nodes. - #[pallet::storage] - pub(crate) type VoterBags = - StorageMap<_, Twox64Concat, VoteWeight, voter_bags::Bag>; - - /// Voter nodes store links forward and back within their respective bags, the stash id, and - /// whether the voter is a validator or nominator. - /// - /// There is nothing in this map directly identifying to which bag a particular node belongs. - /// However, the `Node` data structure has helpers which can provide that information. - #[pallet::storage] - pub(crate) type VoterNodes = - StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::Node>; - - // End of voter bags data. - - /// The threshold for when users can start calling `chill_other` for other validators / nominators. - /// The threshold is compared to the actual number of validators / nominators (`CountFor*`) in - /// the system compared to the configured max (`Max*Count`). - #[pallet::storage] - pub(crate) type ChillThreshold = StorageValue<_, Percent, OptionQuery>; - - #[pallet::genesis_config] - pub struct GenesisConfig { - pub history_depth: u32, - pub validator_count: u32, - pub minimum_validator_count: u32, - pub invulnerables: Vec, - pub force_era: Forcing, - pub slash_reward_fraction: Perbill, - pub canceled_payout: BalanceOf, - pub stakers: Vec<(T::AccountId, T::AccountId, BalanceOf, StakerStatus)>, - pub min_nominator_bond: BalanceOf, - pub min_validator_bond: BalanceOf, - } - - #[cfg(feature = "std")] - impl Default for GenesisConfig { - fn default() -> Self { - GenesisConfig { - history_depth: 84u32, - validator_count: Default::default(), - minimum_validator_count: Default::default(), - invulnerables: Default::default(), - force_era: Default::default(), - slash_reward_fraction: Default::default(), - canceled_payout: Default::default(), - stakers: Default::default(), - min_nominator_bond: Default::default(), - min_validator_bond: Default::default(), - } - } - } - - #[pallet::genesis_build] - impl GenesisBuild for GenesisConfig { - fn build(&self) { - HistoryDepth::::put(self.history_depth); - ValidatorCount::::put(self.validator_count); - MinimumValidatorCount::::put(self.minimum_validator_count); - Invulnerables::::put(&self.invulnerables); - ForceEra::::put(self.force_era); - CanceledSlashPayout::::put(self.canceled_payout); - SlashRewardFraction::::put(self.slash_reward_fraction); - StorageVersion::::put(Releases::V7_0_0); - MinNominatorBond::::put(self.min_nominator_bond); - MinValidatorBond::::put(self.min_validator_bond); - - let mut num_voters: u32 = 0; - for &(ref stash, ref controller, balance, ref status) in &self.stakers { - log!( - trace, - "inserting genesis staker: {:?} => {:?} => {:?}", - stash, - balance, - status - ); - assert!( - T::Currency::free_balance(&stash) >= balance, - "Stash does not have enough balance to bond." - ); - - if let Err(why) = >::bond( - T::Origin::from(Some(stash.clone()).into()), - T::Lookup::unlookup(controller.clone()), - balance, - RewardDestination::Staked, - ) { - // TODO: later on, fix all the tests that trigger these warnings, and - // make these assertions. Genesis stakers should all be correct! - log!(warn, "failed to bond staker at genesis: {:?}.", why); - continue - } - match status { - StakerStatus::Validator => { - if let Err(why) = >::validate( - T::Origin::from(Some(controller.clone()).into()), - Default::default(), - ) { - log!(warn, "failed to validate staker at genesis: {:?}.", why); - } else { - num_voters += 1; - } - }, - StakerStatus::Nominator(votes) => { - if let Err(why) = >::nominate( - T::Origin::from(Some(controller.clone()).into()), - votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), - ) { - log!(warn, "failed to nominate staker at genesis: {:?}.", why); - } else { - num_voters += 1; - } - }, - _ => (), - }; - } - - // all voters are inserted sanely. - assert_eq!( - CounterForVoters::::get(), - num_voters, - "not all genesis stakers were inserted into bags, something is wrong." - ); - } - } - - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - #[pallet::metadata(T::AccountId = "AccountId", BalanceOf = "Balance")] - pub enum Event { - /// The era payout has been set; the first balance is the validator-payout; the second is - /// the remainder from the maximum amount of reward. - /// \[era_index, validator_payout, remainder\] - EraPayout(EraIndex, BalanceOf, BalanceOf), - /// The staker has been rewarded by this amount. \[stash, amount\] - Reward(T::AccountId, BalanceOf), - /// One validator (and its nominators) has been slashed by the given amount. - /// \[validator, amount\] - Slash(T::AccountId, BalanceOf), - /// An old slashing report from a prior era was discarded because it could - /// not be processed. \[session_index\] - OldSlashingReportDiscarded(SessionIndex), - /// A new set of stakers was elected. - StakingElection, - /// An account has bonded this amount. \[stash, amount\] - /// - /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, - /// it will not be emitted for staking rewards when they are added to stake. - Bonded(T::AccountId, BalanceOf), - /// An account has unbonded this amount. \[stash, amount\] - Unbonded(T::AccountId, BalanceOf), - /// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance` - /// from the unlocking queue. \[stash, amount\] - Withdrawn(T::AccountId, BalanceOf), - /// A nominator has been kicked from a validator. \[nominator, stash\] - Kicked(T::AccountId, T::AccountId), - /// The election failed. No new era is planned. - StakingElectionFailed, - /// An account has stopped participating as either a validator or nominator. - /// \[stash\] - Chilled(T::AccountId), - /// Moved an account from one bag to another. \[who, from, to\]. - Rebagged(T::AccountId, VoteWeight, VoteWeight), - } - - #[pallet::error] - pub enum Error { - /// Not a controller account. - NotController, - /// Not a stash account. - NotStash, - /// Stash is already bonded. - AlreadyBonded, - /// Controller is already paired. - AlreadyPaired, - /// Targets cannot be empty. - EmptyTargets, - /// Duplicate index. - DuplicateIndex, - /// Slash record index out of bounds. - InvalidSlashIndex, - /// Can not bond with value less than minimum required. - InsufficientBond, - /// Can not schedule more unlock chunks. - NoMoreChunks, - /// Can not rebond without unlocking chunks. - NoUnlockChunk, - /// Attempting to target a stash that still has funds. - FundedTarget, - /// Invalid era to reward. - InvalidEraToReward, - /// Invalid number of nominations. - InvalidNumberOfNominations, - /// Items are not sorted and unique. - NotSortedAndUnique, - /// Rewards for this era have already been claimed for this validator. - AlreadyClaimed, - /// Incorrect previous history depth input provided. - IncorrectHistoryDepth, - /// Incorrect number of slashing spans provided. - IncorrectSlashingSpans, - /// Internal state has become somehow corrupted and the operation cannot continue. - BadState, - /// Too many nomination targets supplied. - TooManyTargets, - /// A nomination target was supplied that was blocked or otherwise not a validator. - BadTarget, - /// The user has enough bond and thus cannot be chilled forcefully by an external person. - CannotChillOther, - /// There are too many nominators in the system. Governance needs to adjust the staking settings - /// to keep things safe for the runtime. - TooManyNominators, - /// There are too many validators in the system. Governance needs to adjust the staking settings - /// to keep things safe for the runtime. - TooManyValidators, - } - - #[pallet::hooks] - impl Hooks> for Pallet { - fn on_runtime_upgrade() -> Weight { - if StorageVersion::::get() == Releases::V6_0_0 { - migrations::v7::migrate::() - } else { - T::DbWeight::get().reads(1) - } - } - - #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result<(), &'static str> { - if StorageVersion::::get() == Releases::V6_0_0 { - migrations::v7::pre_migrate::() - } else { - Ok(()) - } - } - - fn on_initialize(_now: BlockNumberFor) -> Weight { - // just return the weight of the on_finalize. - T::DbWeight::get().reads(1) - } - - fn on_finalize(_n: BlockNumberFor) { - // Set the start of the first era. - if let Some(mut active_era) = Self::active_era() { - if active_era.start.is_none() { - let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); - active_era.start = Some(now_as_millis_u64); - // This write only ever happens once, we don't include it in the weight in general - ActiveEra::::put(active_era); - } - } - // `on_finalize` weight is tracked in `on_initialize` - } - - fn integrity_test() { - sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| { - assert!( - T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, - "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", - T::SlashDeferDuration::get(), - T::BondingDuration::get(), - ); - - assert!( - T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "Voter bag thresholds must strictly increase", - ); - - assert!( - { - let existential_weight = voter_bags::existential_weight::(); - T::VoterBagThresholds::get() - .first() - .map(|&lowest_threshold| lowest_threshold >= existential_weight) - .unwrap_or(true) - }, - "Smallest bag should not be smaller than existential weight", - ); - }); - } - } - } - - #[pallet::call] - impl Pallet { - /// Take the origin account as a stash and lock up `value` of its balance. `controller` will - /// be the account that controls it. - /// - /// `value` must be more than the `minimum_balance` specified by `T::Currency`. - /// - /// The dispatch origin for this call must be _Signed_ by the stash account. - /// - /// Emits `Bonded`. - /// # - /// - Independent of the arguments. Moderate complexity. - /// - O(1). - /// - Three extra DB entries. - /// - /// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned - /// unless the `origin` falls below _existential deposit_ and gets removed as dust. - /// ------------------ - /// # - #[pallet::weight(T::WeightInfo::bond())] - pub fn bond( - origin: OriginFor, - controller: ::Source, - #[pallet::compact] value: BalanceOf, - payee: RewardDestination, - ) -> DispatchResult { - let stash = ensure_signed(origin)?; - - if >::contains_key(&stash) { - Err(Error::::AlreadyBonded)? - } - - let controller = T::Lookup::lookup(controller)?; - - if >::contains_key(&controller) { - Err(Error::::AlreadyPaired)? - } - - // Reject a bond which is considered to be _dust_. - if value < T::Currency::minimum_balance() { - Err(Error::::InsufficientBond)? - } - - frame_system::Pallet::::inc_consumers(&stash).map_err(|_| Error::::BadState)?; - - // You're auto-bonded forever, here. We might improve this by only bonding when - // you actually validate/nominate and remove once you unbond __everything__. - >::insert(&stash, &controller); - >::insert(&stash, payee); - - let current_era = CurrentEra::::get().unwrap_or(0); - let history_depth = Self::history_depth(); - let last_reward_era = current_era.saturating_sub(history_depth); - - let stash_balance = T::Currency::free_balance(&stash); - let value = value.min(stash_balance); - Self::deposit_event(Event::::Bonded(stash.clone(), value)); - let item = StakingLedger { - stash, - total: value, - active: value, - unlocking: vec![], - claimed_rewards: (last_reward_era..current_era).collect(), - }; - Self::update_ledger(&controller, &item); - Ok(()) - } - - /// Add some extra amount that have appeared in the stash `free_balance` into the balance up - /// for staking. - /// - /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. - /// - /// Use this if there are additional funds in your stash account that you wish to bond. - /// Unlike [`bond`](Self::bond) or [`unbond`](Self::unbond) this function does not impose any limitation - /// on the amount that can be added. - /// - /// Emits `Bonded`. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - O(1). - /// # - #[pallet::weight(T::WeightInfo::bond_extra())] - pub fn bond_extra( - origin: OriginFor, - #[pallet::compact] max_additional: BalanceOf, - ) -> DispatchResult { - let stash = ensure_signed(origin)?; - - let controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; - let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - - let stash_balance = T::Currency::free_balance(&stash); - if let Some(extra) = stash_balance.checked_sub(&ledger.total) { - let extra = extra.min(max_additional); - ledger.total += extra; - ledger.active += extra; - // Last check: the new active amount of ledger must be more than ED. - ensure!( - ledger.active >= T::Currency::minimum_balance(), - Error::::InsufficientBond - ); - - Self::deposit_event(Event::::Bonded(stash.clone(), extra)); - Self::update_ledger(&controller, &ledger); - Self::do_rebag(&stash); - } - Ok(()) - } - - /// Schedule a portion of the stash to be unlocked ready for transfer out after the bond - /// period ends. If this leaves an amount actively bonded less than - /// T::Currency::minimum_balance(), then it is increased to the full amount. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// Once the unlock period is done, you can call `withdraw_unbonded` to actually move - /// the funds out of management ready for transfer. - /// - /// No more than a limited number of unlocking chunks (see `MAX_UNLOCKING_CHUNKS`) - /// can co-exists at the same time. In that case, [`Call::withdraw_unbonded`] need - /// to be called first to remove some of the chunks (if possible). - /// - /// If a user encounters the `InsufficientBond` error when calling this extrinsic, - /// they should call `chill` first in order to free up their bonded funds. - /// - /// Emits `Unbonded`. - /// - /// See also [`Call::withdraw_unbonded`]. - #[pallet::weight(T::WeightInfo::unbond())] - pub fn unbond( - origin: OriginFor, - #[pallet::compact] value: BalanceOf, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks); - - let mut value = value.min(ledger.active); - - if !value.is_zero() { - ledger.active -= value; - - // Avoid there being a dust balance left in the staking system. - if ledger.active < T::Currency::minimum_balance() { - value += ledger.active; - ledger.active = Zero::zero(); - } - - let min_active_bond = if Nominators::::contains_key(&ledger.stash) { - MinNominatorBond::::get() - } else if Validators::::contains_key(&ledger.stash) { - MinValidatorBond::::get() - } else { - Zero::zero() - }; - - // Make sure that the user maintains enough active bond for their role. - // If a user runs into this error, they should chill first. - ensure!(ledger.active >= min_active_bond, Error::::InsufficientBond); - - // Note: in case there is no current era it is fine to bond one era more. - let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); - ledger.unlocking.push(UnlockChunk { value, era }); - Self::update_ledger(&controller, &ledger); - Self::do_rebag(&ledger.stash); - Self::deposit_event(Event::::Unbonded(ledger.stash, value)); - } - Ok(()) - } - - /// Remove any unlocked chunks from the `unlocking` queue from our management. - /// - /// This essentially frees up that balance to be used by the stash account to do - /// whatever it wants. - /// - /// The dispatch origin for this call must be _Signed_ by the controller. - /// - /// Emits `Withdrawn`. - /// - /// See also [`Call::unbond`]. - /// - /// # - /// Complexity O(S) where S is the number of slashing spans to remove - /// NOTE: Weight annotation is the kill scenario, we refund otherwise. - /// # - #[pallet::weight(T::WeightInfo::withdraw_unbonded_kill(*num_slashing_spans))] - pub fn withdraw_unbonded( - origin: OriginFor, - num_slashing_spans: u32, - ) -> DispatchResultWithPostInfo { - let controller = ensure_signed(origin)?; - let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let (stash, old_total) = (ledger.stash.clone(), ledger.total); - if let Some(current_era) = Self::current_era() { - ledger = ledger.consolidate_unlocked(current_era) - } - - let post_info_weight = - if ledger.unlocking.is_empty() && ledger.active < T::Currency::minimum_balance() { - // This account must have called `unbond()` with some value that caused the active - // portion to fall below existential deposit + will have no more unlocking chunks - // left. We can now safely remove all staking-related information. - Self::kill_stash(&stash, num_slashing_spans)?; - // Remove the lock. - T::Currency::remove_lock(STAKING_ID, &stash); - // This is worst case scenario, so we use the full weight and return None - None - } else { - // This was the consequence of a partial unbond. just update the ledger and move on. - Self::update_ledger(&controller, &ledger); - - // This is only an update, so we use less overall weight. - Some(T::WeightInfo::withdraw_unbonded_update(num_slashing_spans)) - }; - - // `old_total` should never be less than the new total because - // `consolidate_unlocked` strictly subtracts balance. - if ledger.total < old_total { - // Already checked that this won't overflow by entry condition. - let value = old_total - ledger.total; - Self::deposit_event(Event::::Withdrawn(stash, value)); - } - - Ok(post_info_weight.into()) - } - - /// Declare the desire to validate for the origin controller. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - #[pallet::weight(T::WeightInfo::validate())] - pub fn validate(origin: OriginFor, prefs: ValidatorPrefs) -> DispatchResult { - let controller = ensure_signed(origin)?; - - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); - let stash = &ledger.stash; - - // Only check limits if they are not already a validator. - if !Validators::::contains_key(stash) { - // If this error is reached, we need to adjust the `MinValidatorBond` and start calling `chill_other`. - // Until then, we explicitly block new validators to protect the runtime. - if let Some(max_validators) = MaxValidatorsCount::::get() { - ensure!( - CounterForValidators::::get() < max_validators, - Error::::TooManyValidators - ); - } - } - - Self::do_remove_nominator(stash); - Self::do_add_validator(stash, prefs); - Ok(()) - } - - /// Declare the desire to nominate `targets` for the origin controller. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// # - /// - The transaction's complexity is proportional to the size of `targets` (N) - /// which is capped at CompactAssignments::LIMIT (MAX_NOMINATIONS). - /// - Both the reads and writes follow a similar pattern. - /// # - #[pallet::weight(T::WeightInfo::nominate(targets.len() as u32))] - pub fn nominate( - origin: OriginFor, - targets: Vec<::Source>, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.active >= MinNominatorBond::::get(), Error::::InsufficientBond); - let stash = &ledger.stash; - - // Only check limits if they are not already a nominator. - if !Nominators::::contains_key(stash) { - // If this error is reached, we need to adjust the `MinNominatorBond` and start calling `chill_other`. - // Until then, we explicitly block new nominators to protect the runtime. - if let Some(max_nominators) = MaxNominatorsCount::::get() { - ensure!( - CounterForNominators::::get() < max_nominators, - Error::::TooManyNominators - ); - } - } - - ensure!(!targets.is_empty(), Error::::EmptyTargets); - ensure!(targets.len() <= T::MAX_NOMINATIONS as usize, Error::::TooManyTargets); - - let old = Nominators::::get(stash).map_or_else(Vec::new, |x| x.targets); - - let targets = targets - .into_iter() - .map(|t| T::Lookup::lookup(t).map_err(DispatchError::from)) - .map(|n| { - n.and_then(|n| { - if old.contains(&n) || !Validators::::get(&n).blocked { - Ok(n) - } else { - Err(Error::::BadTarget.into()) - } - }) - }) - .collect::, _>>()?; - - let nominations = Nominations { - targets, - // Initial nominations are considered submitted at era 0. See `Nominations` doc - submitted_in: Self::current_era().unwrap_or(0), - suppressed: false, - }; - - Self::do_remove_validator(stash); - Self::do_add_nominator(stash, nominations); - Ok(()) - } - - /// Declare no desire to either validate or nominate. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - Contains one read. - /// - Writes are limited to the `origin` account key. - /// # - #[pallet::weight(T::WeightInfo::chill())] - pub fn chill(origin: OriginFor) -> DispatchResult { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - Self::chill_stash(&ledger.stash); - Ok(()) - } - - /// (Re-)set the payment target for a controller. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - Contains a limited number of reads. - /// - Writes are limited to the `origin` account key. - /// --------- - /// - Weight: O(1) - /// - DB Weight: - /// - Read: Ledger - /// - Write: Payee - /// # - #[pallet::weight(T::WeightInfo::set_payee())] - pub fn set_payee( - origin: OriginFor, - payee: RewardDestination, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let stash = &ledger.stash; - >::insert(stash, payee); - Ok(()) - } - - /// (Re-)set the controller of a stash. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. - /// - /// # - /// - Independent of the arguments. Insignificant complexity. - /// - Contains a limited number of reads. - /// - Writes are limited to the `origin` account key. - /// ---------- - /// Weight: O(1) - /// DB Weight: - /// - Read: Bonded, Ledger New Controller, Ledger Old Controller - /// - Write: Bonded, Ledger New Controller, Ledger Old Controller - /// # - #[pallet::weight(T::WeightInfo::set_controller())] - pub fn set_controller( - origin: OriginFor, - controller: ::Source, - ) -> DispatchResult { - let stash = ensure_signed(origin)?; - let old_controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; - let controller = T::Lookup::lookup(controller)?; - if >::contains_key(&controller) { - Err(Error::::AlreadyPaired)? - } - if controller != old_controller { - >::insert(&stash, &controller); - if let Some(l) = >::take(&old_controller) { - >::insert(&controller, l); - } - } - Ok(()) - } - - /// Sets the ideal number of validators. - /// - /// The dispatch origin must be Root. - /// - /// # - /// Weight: O(1) - /// Write: Validator Count - /// # - #[pallet::weight(T::WeightInfo::set_validator_count())] - pub fn set_validator_count( - origin: OriginFor, - #[pallet::compact] new: u32, - ) -> DispatchResult { - ensure_root(origin)?; - ValidatorCount::::put(new); - Ok(()) - } - - /// Increments the ideal number of validators. - /// - /// The dispatch origin must be Root. - /// - /// # - /// Same as [`Self::set_validator_count`]. - /// # - #[pallet::weight(T::WeightInfo::set_validator_count())] - pub fn increase_validator_count( - origin: OriginFor, - #[pallet::compact] additional: u32, - ) -> DispatchResult { - ensure_root(origin)?; - ValidatorCount::::mutate(|n| *n += additional); - Ok(()) - } - - /// Scale up the ideal number of validators by a factor. - /// - /// The dispatch origin must be Root. - /// - /// # - /// Same as [`Self::set_validator_count`]. - /// # - #[pallet::weight(T::WeightInfo::set_validator_count())] - pub fn scale_validator_count(origin: OriginFor, factor: Percent) -> DispatchResult { - ensure_root(origin)?; - ValidatorCount::::mutate(|n| *n += factor * *n); - Ok(()) - } - - /// Force there to be no new eras indefinitely. - /// - /// The dispatch origin must be Root. - /// - /// # Warning - /// - /// The election process starts multiple blocks before the end of the era. - /// Thus the election process may be ongoing when this is called. In this case the - /// election will continue until the next era is triggered. - /// - /// # - /// - No arguments. - /// - Weight: O(1) - /// - Write: ForceEra - /// # - #[pallet::weight(T::WeightInfo::force_no_eras())] - pub fn force_no_eras(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - ForceEra::::put(Forcing::ForceNone); - Ok(()) - } - - /// Force there to be a new era at the end of the next session. After this, it will be - /// reset to normal (non-forced) behaviour. - /// - /// The dispatch origin must be Root. - /// - /// # Warning - /// - /// The election process starts multiple blocks before the end of the era. - /// If this is called just before a new era is triggered, the election process may not - /// have enough blocks to get a result. - /// - /// # - /// - No arguments. - /// - Weight: O(1) - /// - Write ForceEra - /// # - #[pallet::weight(T::WeightInfo::force_new_era())] - pub fn force_new_era(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - ForceEra::::put(Forcing::ForceNew); - Ok(()) - } - - /// Set the validators who cannot be slashed (if any). - /// - /// The dispatch origin must be Root. - /// - /// # - /// - O(V) - /// - Write: Invulnerables - /// # - #[pallet::weight(T::WeightInfo::set_invulnerables(invulnerables.len() as u32))] - pub fn set_invulnerables( - origin: OriginFor, - invulnerables: Vec, - ) -> DispatchResult { - ensure_root(origin)?; - >::put(invulnerables); - Ok(()) - } - - /// Force a current staker to become completely unstaked, immediately. - /// - /// The dispatch origin must be Root. - /// - /// # - /// O(S) where S is the number of slashing spans to be removed - /// Reads: Bonded, Slashing Spans, Account, Locks - /// Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Account, Locks - /// Writes Each: SpanSlash * S - /// # - #[pallet::weight(T::WeightInfo::force_unstake(*num_slashing_spans))] - pub fn force_unstake( - origin: OriginFor, - stash: T::AccountId, - num_slashing_spans: u32, - ) -> DispatchResult { - ensure_root(origin)?; - - // Remove all staking-related information. - Self::kill_stash(&stash, num_slashing_spans)?; - - // Remove the lock. - T::Currency::remove_lock(STAKING_ID, &stash); - Ok(()) - } - - /// Force there to be a new era at the end of sessions indefinitely. - /// - /// The dispatch origin must be Root. - /// - /// # Warning - /// - /// The election process starts multiple blocks before the end of the era. - /// If this is called just before a new era is triggered, the election process may not - /// have enough blocks to get a result. - /// - /// # - /// - Weight: O(1) - /// - Write: ForceEra - /// # - #[pallet::weight(T::WeightInfo::force_new_era_always())] - pub fn force_new_era_always(origin: OriginFor) -> DispatchResult { - ensure_root(origin)?; - ForceEra::::put(Forcing::ForceAlways); - Ok(()) - } - - /// Cancel enactment of a deferred slash. - /// - /// Can be called by the `T::SlashCancelOrigin`. - /// - /// Parameters: era and indices of the slashes for that era to kill. - /// - /// # - /// Complexity: O(U + S) - /// with U unapplied slashes weighted with U=1000 - /// and S is the number of slash indices to be canceled. - /// - Read: Unapplied Slashes - /// - Write: Unapplied Slashes - /// # - #[pallet::weight(T::WeightInfo::cancel_deferred_slash(slash_indices.len() as u32))] - pub fn cancel_deferred_slash( - origin: OriginFor, - era: EraIndex, - slash_indices: Vec, - ) -> DispatchResult { - T::SlashCancelOrigin::ensure_origin(origin)?; - - ensure!(!slash_indices.is_empty(), Error::::EmptyTargets); - ensure!(is_sorted_and_unique(&slash_indices), Error::::NotSortedAndUnique); - - let mut unapplied = ::UnappliedSlashes::get(&era); - let last_item = slash_indices[slash_indices.len() - 1]; - ensure!((last_item as usize) < unapplied.len(), Error::::InvalidSlashIndex); - - for (removed, index) in slash_indices.into_iter().enumerate() { - let index = (index as usize) - removed; - unapplied.remove(index); - } - - ::UnappliedSlashes::insert(&era, &unapplied); - Ok(()) - } - - /// Pay out all the stakers behind a single validator for a single era. - /// - /// - `validator_stash` is the stash account of the validator. Their nominators, up to - /// `T::MaxNominatorRewardedPerValidator`, will also receive their rewards. - /// - `era` may be any era between `[current_era - history_depth; current_era]`. - /// - /// The origin of this call must be _Signed_. Any account can call this function, even if - /// it is not one of the stakers. - /// - /// # - /// - Time complexity: at most O(MaxNominatorRewardedPerValidator). - /// - Contains a limited number of reads and writes. - /// ----------- - /// N is the Number of payouts for the validator (including the validator) - /// Weight: - /// - Reward Destination Staked: O(N) - /// - Reward Destination Controller (Creating): O(N) - /// - /// NOTE: weights are assuming that payouts are made to alive stash account (Staked). - /// Paying even a dead controller is cheaper weight-wise. We don't do any refunds here. - /// # - #[pallet::weight(T::WeightInfo::payout_stakers_alive_staked( - T::MaxNominatorRewardedPerValidator::get() - ))] - pub fn payout_stakers( - origin: OriginFor, - validator_stash: T::AccountId, - era: EraIndex, - ) -> DispatchResultWithPostInfo { - ensure_signed(origin)?; - Self::do_payout_stakers(validator_stash, era) - } - - /// Rebond a portion of the stash scheduled to be unlocked. - /// - /// The dispatch origin must be signed by the controller. - /// - /// # - /// - Time complexity: O(L), where L is unlocking chunks - /// - Bounded by `MAX_UNLOCKING_CHUNKS`. - /// - Storage changes: Can't increase storage, only decrease it. - /// # - #[pallet::weight(T::WeightInfo::rebond(MAX_UNLOCKING_CHUNKS as u32))] - pub fn rebond( - origin: OriginFor, - #[pallet::compact] value: BalanceOf, - ) -> DispatchResultWithPostInfo { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(!ledger.unlocking.is_empty(), Error::::NoUnlockChunk); - - let ledger = ledger.rebond(value); - // Last check: the new active amount of ledger must be more than ED. - ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); - - Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); - Self::update_ledger(&controller, &ledger); - Self::do_rebag(&ledger.stash); - Ok(Some( - 35 * WEIGHT_PER_MICROS + - 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + - T::DbWeight::get().reads_writes(3, 2), - ) - .into()) - } - - /// Set `HistoryDepth` value. This function will delete any history information - /// when `HistoryDepth` is reduced. - /// - /// Parameters: - /// - `new_history_depth`: The new history depth you would like to set. - /// - `era_items_deleted`: The number of items that will be deleted by this dispatch. - /// This should report all the storage items that will be deleted by clearing old - /// era history. Needed to report an accurate weight for the dispatch. Trusted by - /// `Root` to report an accurate number. - /// - /// Origin must be root. - /// - /// # - /// - E: Number of history depths removed, i.e. 10 -> 7 = 3 - /// - Weight: O(E) - /// - DB Weight: - /// - Reads: Current Era, History Depth - /// - Writes: History Depth - /// - Clear Prefix Each: Era Stakers, EraStakersClipped, ErasValidatorPrefs - /// - Writes Each: ErasValidatorReward, ErasRewardPoints, ErasTotalStake, ErasStartSessionIndex - /// # - #[pallet::weight(T::WeightInfo::set_history_depth(*_era_items_deleted))] - pub fn set_history_depth( - origin: OriginFor, - #[pallet::compact] new_history_depth: EraIndex, - #[pallet::compact] _era_items_deleted: u32, - ) -> DispatchResult { - ensure_root(origin)?; - if let Some(current_era) = Self::current_era() { - HistoryDepth::::mutate(|history_depth| { - let last_kept = current_era.checked_sub(*history_depth).unwrap_or(0); - let new_last_kept = current_era.checked_sub(new_history_depth).unwrap_or(0); - for era_index in last_kept..new_last_kept { - Self::clear_era_information(era_index); - } - *history_depth = new_history_depth - }) - } - Ok(()) - } - - /// Remove all data structure concerning a staker/stash once its balance is at the minimum. - /// This is essentially equivalent to `withdraw_unbonded` except it can be called by anyone - /// and the target `stash` must have no funds left beyond the ED. - /// - /// This can be called from any origin. - /// - /// - `stash`: The stash account to reap. Its balance must be zero. - /// - /// # - /// Complexity: O(S) where S is the number of slashing spans on the account. - /// DB Weight: - /// - Reads: Stash Account, Bonded, Slashing Spans, Locks - /// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Stash Account, Locks - /// - Writes Each: SpanSlash * S - /// # - #[pallet::weight(T::WeightInfo::reap_stash(*num_slashing_spans))] - pub fn reap_stash( - _origin: OriginFor, - stash: T::AccountId, - num_slashing_spans: u32, - ) -> DispatchResult { - let at_minimum = T::Currency::total_balance(&stash) == T::Currency::minimum_balance(); - ensure!(at_minimum, Error::::FundedTarget); - Self::kill_stash(&stash, num_slashing_spans)?; - T::Currency::remove_lock(STAKING_ID, &stash); - Ok(()) - } - - /// Remove the given nominations from the calling validator. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. - /// - /// - `who`: A list of nominator stash accounts who are nominating this validator which - /// should no longer be nominating this validator. - /// - /// Note: Making this call only makes sense if you first set the validator preferences to - /// block any further nominations. - #[pallet::weight(T::WeightInfo::kick(who.len() as u32))] - pub fn kick( - origin: OriginFor, - who: Vec<::Source>, - ) -> DispatchResult { - let controller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let stash = &ledger.stash; - - for nom_stash in who - .into_iter() - .map(T::Lookup::lookup) - .collect::, _>>()? - .into_iter() - { - Nominators::::mutate(&nom_stash, |maybe_nom| { - if let Some(ref mut nom) = maybe_nom { - if let Some(pos) = nom.targets.iter().position(|v| v == stash) { - nom.targets.swap_remove(pos); - Self::deposit_event(Event::::Kicked( - nom_stash.clone(), - stash.clone(), - )); - } - } - }); - } - - Ok(()) - } - - /// Update the various staking limits this pallet. - /// - /// * `min_nominator_bond`: The minimum active bond needed to be a nominator. - /// * `min_validator_bond`: The minimum active bond needed to be a validator. - /// * `max_nominator_count`: The max number of users who can be a nominator at once. - /// When set to `None`, no limit is enforced. - /// * `max_validator_count`: The max number of users who can be a validator at once. - /// When set to `None`, no limit is enforced. - /// - /// Origin must be Root to call this function. - /// - /// NOTE: Existing nominators and validators will not be affected by this update. - /// to kick people under the new limits, `chill_other` should be called. - #[pallet::weight(T::WeightInfo::set_staking_limits())] - pub fn set_staking_limits( - origin: OriginFor, - min_nominator_bond: BalanceOf, - min_validator_bond: BalanceOf, - max_nominator_count: Option, - max_validator_count: Option, - threshold: Option, - ) -> DispatchResult { - ensure_root(origin)?; - MinNominatorBond::::set(min_nominator_bond); - MinValidatorBond::::set(min_validator_bond); - MaxNominatorsCount::::set(max_nominator_count); - MaxValidatorsCount::::set(max_validator_count); - ChillThreshold::::set(threshold); - Ok(()) - } - - /// Declare a `controller` to stop participating as either a validator or nominator. - /// - /// Effects will be felt at the beginning of the next era. - /// - /// The dispatch origin for this call must be _Signed_, but can be called by anyone. - /// - /// If the caller is the same as the controller being targeted, then no further checks are - /// enforced, and this function behaves just like `chill`. - /// - /// If the caller is different than the controller being targeted, the following conditions - /// must be met: - /// * A `ChillThreshold` must be set and checked which defines how close to the max - /// nominators or validators we must reach before users can start chilling one-another. - /// * A `MaxNominatorCount` and `MaxValidatorCount` must be set which is used to determine - /// how close we are to the threshold. - /// * A `MinNominatorBond` and `MinValidatorBond` must be set and checked, which determines - /// if this is a person that should be chilled because they have not met the threshold - /// bond required. - /// - /// This can be helpful if bond requirements are updated, and we need to remove old users - /// who do not satisfy these requirements. - // TODO: Maybe we can deprecate `chill` in the future. - // https://github.com/paritytech/substrate/issues/9111 - #[pallet::weight(T::WeightInfo::chill_other())] - pub fn chill_other(origin: OriginFor, controller: T::AccountId) -> DispatchResult { - // Anyone can call this function. - let caller = ensure_signed(origin)?; - let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - let stash = ledger.stash; - - // In order for one user to chill another user, the following conditions must be met: - // * A `ChillThreshold` is set which defines how close to the max nominators or - // validators we must reach before users can start chilling one-another. - // * A `MaxNominatorCount` and `MaxValidatorCount` which is used to determine how close - // we are to the threshold. - // * A `MinNominatorBond` and `MinValidatorBond` which is the final condition checked to - // determine this is a person that should be chilled because they have not met the - // threshold bond required. - // - // Otherwise, if caller is the same as the controller, this is just like `chill`. - if caller != controller { - let threshold = ChillThreshold::::get().ok_or(Error::::CannotChillOther)?; - let min_active_bond = if Nominators::::contains_key(&stash) { - let max_nominator_count = - MaxNominatorsCount::::get().ok_or(Error::::CannotChillOther)?; - let current_nominator_count = CounterForNominators::::get(); - ensure!( - threshold * max_nominator_count < current_nominator_count, - Error::::CannotChillOther - ); - MinNominatorBond::::get() - } else if Validators::::contains_key(&stash) { - let max_validator_count = - MaxValidatorsCount::::get().ok_or(Error::::CannotChillOther)?; - let current_validator_count = CounterForValidators::::get(); - ensure!( - threshold * max_validator_count < current_validator_count, - Error::::CannotChillOther - ); - MinValidatorBond::::get() - } else { - Zero::zero() - }; - - ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); - } - - Self::chill_stash(&stash); - Ok(()) - } - - /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its - /// stake that it should properly fall into a different bag than its current position. - /// - /// This will adjust its position into the appropriate bag. This will affect its position - /// among the nominator/validator set once the snapshot is prepared for the election. - /// - /// Anyone can call this function about any stash. - #[pallet::weight(T::WeightInfo::rebag())] - pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { - ensure_signed(origin)?; - Pallet::::do_rebag(&stash); - Ok(()) - } - } -} - -impl Pallet { - /// The total balance that can be slashed from a stash account as of right now. - pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { - // Weight note: consider making the stake accessible through stash. - Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() - } - - /// Internal impl of [`Self::slashable_balance_of`] that returns [`VoteWeight`]. - pub fn slashable_balance_of_vote_weight( - stash: &T::AccountId, - issuance: BalanceOf, - ) -> VoteWeight { - T::CurrencyToVote::to_vote(Self::slashable_balance_of(stash), issuance) - } - - /// Returns a closure around `slashable_balance_of_vote_weight` that can be passed around. - /// - /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is - /// important to be only used while the total issuance is not changing. - pub fn weight_of_fn() -> Box VoteWeight> { - // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still - // compile, while some types in mock fail to resolve. - let issuance = T::Currency::total_issuance(); - Box::new(move |who: &T::AccountId| -> VoteWeight { - Self::slashable_balance_of_vote_weight(who, issuance) - }) - } - - fn do_payout_stakers( - validator_stash: T::AccountId, - era: EraIndex, - ) -> DispatchResultWithPostInfo { - // Validate input data - let current_era = CurrentEra::::get().ok_or_else(|| { - Error::::InvalidEraToReward - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - })?; - let history_depth = Self::history_depth(); - ensure!( - era <= current_era && era >= current_era.saturating_sub(history_depth), - Error::::InvalidEraToReward - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - ); - - // Note: if era has no reward to be claimed, era may be future. better not to update - // `ledger.claimed_rewards` in this case. - let era_payout = >::get(&era).ok_or_else(|| { - Error::::InvalidEraToReward - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - })?; - - let controller = Self::bonded(&validator_stash).ok_or_else(|| { - Error::::NotStash.with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) - })?; - let mut ledger = >::get(&controller).ok_or(Error::::NotController)?; - - ledger - .claimed_rewards - .retain(|&x| x >= current_era.saturating_sub(history_depth)); - match ledger.claimed_rewards.binary_search(&era) { - Ok(_) => Err(Error::::AlreadyClaimed - .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)))?, - Err(pos) => ledger.claimed_rewards.insert(pos, era), - } - - let exposure = >::get(&era, &ledger.stash); - - // Input data seems good, no errors allowed after this point - - >::insert(&controller, &ledger); - - // Get Era reward points. It has TOTAL and INDIVIDUAL - // Find the fraction of the era reward that belongs to the validator - // Take that fraction of the eras rewards to split to nominator and validator - // - // Then look at the validator, figure out the proportion of their reward - // which goes to them and each of their nominators. - - let era_reward_points = >::get(&era); - let total_reward_points = era_reward_points.total; - let validator_reward_points = era_reward_points - .individual - .get(&ledger.stash) - .map(|points| *points) - .unwrap_or_else(|| Zero::zero()); - - // Nothing to do if they have no reward points. - if validator_reward_points.is_zero() { - return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) - } - - // This is the fraction of the total reward that the validator and the - // nominators will get. - let validator_total_reward_part = - Perbill::from_rational(validator_reward_points, total_reward_points); - - // This is how much validator + nominators are entitled to. - let validator_total_payout = validator_total_reward_part * era_payout; - - let validator_prefs = Self::eras_validator_prefs(&era, &validator_stash); - // Validator first gets a cut off the top. - let validator_commission = validator_prefs.commission; - let validator_commission_payout = validator_commission * validator_total_payout; - - let validator_leftover_payout = validator_total_payout - validator_commission_payout; - // Now let's calculate how this is split to the validator. - let validator_exposure_part = Perbill::from_rational(exposure.own, exposure.total); - let validator_staking_payout = validator_exposure_part * validator_leftover_payout; - - // We can now make total validator payout: - if let Some(imbalance) = - Self::make_payout(&ledger.stash, validator_staking_payout + validator_commission_payout) - { - Self::deposit_event(Event::::Reward(ledger.stash, imbalance.peek())); - } - - // Track the number of payout ops to nominators. Note: `WeightInfo::payout_stakers_alive_staked` - // always assumes at least a validator is paid out, so we do not need to count their payout op. - let mut nominator_payout_count: u32 = 0; - - // Lets now calculate how this is split to the nominators. - // Reward only the clipped exposures. Note this is not necessarily sorted. - for nominator in exposure.others.iter() { - let nominator_exposure_part = Perbill::from_rational(nominator.value, exposure.total); - - let nominator_reward: BalanceOf = - nominator_exposure_part * validator_leftover_payout; - // We can now make nominator payout: - if let Some(imbalance) = Self::make_payout(&nominator.who, nominator_reward) { - // Note: this logic does not count payouts for `RewardDestination::None`. - nominator_payout_count += 1; - Self::deposit_event(Event::::Reward(nominator.who.clone(), imbalance.peek())); - } - } - - debug_assert!(nominator_payout_count <= T::MaxNominatorRewardedPerValidator::get()); - Ok(Some(T::WeightInfo::payout_stakers_alive_staked(nominator_payout_count)).into()) - } - - /// Update the ledger for a controller. - /// - /// This will also update the stash lock. - fn update_ledger( - controller: &T::AccountId, - ledger: &StakingLedger>, - ) { - T::Currency::set_lock(STAKING_ID, &ledger.stash, ledger.total, WithdrawReasons::all()); - >::insert(controller, ledger); - } - - /// Chill a stash account. - fn chill_stash(stash: &T::AccountId) { - let chilled_as_validator = Self::do_remove_validator(stash); - let chilled_as_nominator = Self::do_remove_nominator(stash); - if chilled_as_validator || chilled_as_nominator { - Self::deposit_event(Event::::Chilled(stash.clone())); - } - } - - /// Actually make a payment to a staker. This uses the currency's reward function - /// to pay the right payee for the given staker account. - fn make_payout(stash: &T::AccountId, amount: BalanceOf) -> Option> { - let dest = Self::payee(stash); - match dest { - RewardDestination::Controller => Self::bonded(stash) - .and_then(|controller| Some(T::Currency::deposit_creating(&controller, amount))), - RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), - RewardDestination::Staked => Self::bonded(stash) - .and_then(|c| Self::ledger(&c).map(|l| (c, l))) - .and_then(|(controller, mut l)| { - l.active += amount; - l.total += amount; - let r = T::Currency::deposit_into_existing(stash, amount).ok(); - Self::update_ledger(&controller, &l); - r - }), - RewardDestination::Account(dest_account) => - Some(T::Currency::deposit_creating(&dest_account, amount)), - RewardDestination::None => None, - } - } - - /// Plan a new session potentially trigger a new era. - fn new_session(session_index: SessionIndex, is_genesis: bool) -> Option> { - if let Some(current_era) = Self::current_era() { - // Initial era has been set. - let current_era_start_session_index = Self::eras_start_session_index(current_era) - .unwrap_or_else(|| { - frame_support::print("Error: start_session_index must be set for current_era"); - 0 - }); - - let era_length = - session_index.checked_sub(current_era_start_session_index).unwrap_or(0); // Must never happen. - - match ForceEra::::get() { - // Will be set to `NotForcing` again if a new era has been triggered. - Forcing::ForceNew => (), - // Short circuit to `try_trigger_new_era`. - Forcing::ForceAlways => (), - // Only go to `try_trigger_new_era` if deadline reached. - Forcing::NotForcing if era_length >= T::SessionsPerEra::get() => (), - _ => { - // Either `Forcing::ForceNone`, - // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. - return None - }, - } - - // New era. - let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); - if maybe_new_era_validators.is_some() && - matches!(ForceEra::::get(), Forcing::ForceNew) - { - ForceEra::::put(Forcing::NotForcing); - } - - maybe_new_era_validators - } else { - // Set initial era. - log!(debug, "Starting the first era."); - Self::try_trigger_new_era(session_index, is_genesis) - } - } - - /// Start a session potentially starting an era. - fn start_session(start_session: SessionIndex) { - let next_active_era = Self::active_era().map(|e| e.index + 1).unwrap_or(0); - // This is only `Some` when current era has already progressed to the next era, while the - // active era is one behind (i.e. in the *last session of the active era*, or *first session - // of the new current era*, depending on how you look at it). - if let Some(next_active_era_start_session_index) = - Self::eras_start_session_index(next_active_era) - { - if next_active_era_start_session_index == start_session { - Self::start_era(start_session); - } else if next_active_era_start_session_index < start_session { - // This arm should never happen, but better handle it than to stall the staking - // pallet. - frame_support::print("Warning: A session appears to have been skipped."); - Self::start_era(start_session); - } - } - } - - /// End a session potentially ending an era. - fn end_session(session_index: SessionIndex) { - if let Some(active_era) = Self::active_era() { - if let Some(next_active_era_start_session_index) = - Self::eras_start_session_index(active_era.index + 1) - { - if next_active_era_start_session_index == session_index + 1 { - Self::end_era(active_era, session_index); - } - } - } - } - - /// * Increment `active_era.index`, - /// * reset `active_era.start`, - /// * update `BondedEras` and apply slashes. - fn start_era(start_session: SessionIndex) { - let active_era = ActiveEra::::mutate(|active_era| { - let new_index = active_era.as_ref().map(|info| info.index + 1).unwrap_or(0); - *active_era = Some(ActiveEraInfo { - index: new_index, - // Set new active era start in next `on_finalize`. To guarantee usage of `Time` - start: None, - }); - new_index - }); - - let bonding_duration = T::BondingDuration::get(); - - BondedEras::::mutate(|bonded| { - bonded.push((active_era, start_session)); - - if active_era > bonding_duration { - let first_kept = active_era - bonding_duration; - - // Prune out everything that's from before the first-kept index. - let n_to_prune = - bonded.iter().take_while(|&&(era_idx, _)| era_idx < first_kept).count(); - - // Kill slashing metadata. - for (pruned_era, _) in bonded.drain(..n_to_prune) { - slashing::clear_era_metadata::(pruned_era); - } - - if let Some(&(_, first_session)) = bonded.first() { - T::SessionInterface::prune_historical_up_to(first_session); - } - } - }); - - Self::apply_unapplied_slashes(active_era); - } - - /// Compute payout for era. - fn end_era(active_era: ActiveEraInfo, _session_index: SessionIndex) { - // Note: active_era_start can be None if end era is called during genesis config. - if let Some(active_era_start) = active_era.start { - let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); - - let era_duration = (now_as_millis_u64 - active_era_start).saturated_into::(); - let staked = Self::eras_total_stake(&active_era.index); - let issuance = T::Currency::total_issuance(); - let (validator_payout, rest) = T::EraPayout::era_payout(staked, issuance, era_duration); - - Self::deposit_event(Event::::EraPayout(active_era.index, validator_payout, rest)); - - // Set ending era reward. - >::insert(&active_era.index, validator_payout); - T::RewardRemainder::on_unbalanced(T::Currency::issue(rest)); - } - } - - /// Plan a new era. - /// - /// * Bump the current era storage (which holds the latest planned era). - /// * Store start session index for the new planned era. - /// * Clean old era information. - /// * Store staking information for the new planned era - /// - /// Returns the new validator set. - pub fn trigger_new_era( - start_session_index: SessionIndex, - exposures: Vec<(T::AccountId, Exposure>)>, - ) -> Vec { - // Increment or set current era. - let new_planned_era = CurrentEra::::mutate(|s| { - *s = Some(s.map(|s| s + 1).unwrap_or(0)); - s.unwrap() - }); - ErasStartSessionIndex::::insert(&new_planned_era, &start_session_index); - - // Clean old era information. - if let Some(old_era) = new_planned_era.checked_sub(Self::history_depth() + 1) { - Self::clear_era_information(old_era); - } - - // Set staking information for the new era. - Self::store_stakers_info(exposures, new_planned_era) - } - - /// Potentially plan a new era. - /// - /// Get election result from `T::ElectionProvider`. - /// In case election result has more than [`MinimumValidatorCount`] validator trigger a new era. - /// - /// In case a new era is planned, the new validator set is returned. - fn try_trigger_new_era( - start_session_index: SessionIndex, - is_genesis: bool, - ) -> Option> { - let (election_result, weight) = if is_genesis { - T::GenesisElectionProvider::elect().map_err(|e| { - log!(warn, "genesis election provider failed due to {:?}", e); - Self::deposit_event(Event::StakingElectionFailed); - }) - } else { - T::ElectionProvider::elect().map_err(|e| { - log!(warn, "election provider failed due to {:?}", e); - Self::deposit_event(Event::StakingElectionFailed); - }) - } - .ok()?; - - >::register_extra_weight_unchecked( - weight, - frame_support::weights::DispatchClass::Mandatory, - ); - - let exposures = Self::collect_exposures(election_result); - - if (exposures.len() as u32) < Self::minimum_validator_count().max(1) { - // Session will panic if we ever return an empty validator set, thus max(1) ^^. - match CurrentEra::::get() { - Some(current_era) if current_era > 0 => log!( - warn, - "chain does not have enough staking candidates to operate for era {:?} ({} \ - elected, minimum is {})", - CurrentEra::::get().unwrap_or(0), - exposures.len(), - Self::minimum_validator_count(), - ), - None => { - // The initial era is allowed to have no exposures. - // In this case the SessionManager is expected to choose a sensible validator - // set. - // TODO: this should be simplified #8911 - CurrentEra::::put(0); - ErasStartSessionIndex::::insert(&0, &start_session_index); - }, - _ => (), - } - - Self::deposit_event(Event::StakingElectionFailed); - return None - } - - Self::deposit_event(Event::StakingElection); - Some(Self::trigger_new_era(start_session_index, exposures)) - } - - /// Process the output of the election. - /// - /// Store staking information for the new planned era - pub fn store_stakers_info( - exposures: Vec<(T::AccountId, Exposure>)>, - new_planned_era: EraIndex, - ) -> Vec { - let elected_stashes = exposures.iter().cloned().map(|(x, _)| x).collect::>(); - - // Populate stakers, exposures, and the snapshot of validator prefs. - let mut total_stake: BalanceOf = Zero::zero(); - exposures.into_iter().for_each(|(stash, exposure)| { - total_stake = total_stake.saturating_add(exposure.total); - >::insert(new_planned_era, &stash, &exposure); - - let mut exposure_clipped = exposure; - let clipped_max_len = T::MaxNominatorRewardedPerValidator::get() as usize; - if exposure_clipped.others.len() > clipped_max_len { - exposure_clipped.others.sort_by(|a, b| a.value.cmp(&b.value).reverse()); - exposure_clipped.others.truncate(clipped_max_len); - } - >::insert(&new_planned_era, &stash, exposure_clipped); - }); - - // Insert current era staking information - >::insert(&new_planned_era, total_stake); - - // Collect the pref of all winners. - for stash in &elected_stashes { - let pref = Self::validators(stash); - >::insert(&new_planned_era, stash, pref); - } - - if new_planned_era > 0 { - log!( - info, - "new validator set of size {:?} has been processed for era {:?}", - elected_stashes.len(), - new_planned_era, - ); - } - - elected_stashes - } - - /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a - /// [`Exposure`]. - fn collect_exposures( - supports: Supports, - ) -> Vec<(T::AccountId, Exposure>)> { - let total_issuance = T::Currency::total_issuance(); - let to_currency = |e: frame_election_provider_support::ExtendedBalance| { - T::CurrencyToVote::to_currency(e, total_issuance) - }; - - supports - .into_iter() - .map(|(validator, support)| { - // Build `struct exposure` from `support`. - let mut others = Vec::with_capacity(support.voters.len()); - let mut own: BalanceOf = Zero::zero(); - let mut total: BalanceOf = Zero::zero(); - support - .voters - .into_iter() - .map(|(nominator, weight)| (nominator, to_currency(weight))) - .for_each(|(nominator, stake)| { - if nominator == validator { - own = own.saturating_add(stake); - } else { - others.push(IndividualExposure { who: nominator, value: stake }); - } - total = total.saturating_add(stake); - }); - - let exposure = Exposure { own, others, total }; - (validator, exposure) - }) - .collect::)>>() - } - - /// Remove all associated data of a stash account from the staking system. - /// - /// Assumes storage is upgraded before calling. - /// - /// This is called: - /// - after a `withdraw_unbonded()` call that frees all of a stash's bonded balance. - /// - through `reap_stash()` if the balance has fallen to zero (through slashing). - fn kill_stash(stash: &T::AccountId, num_slashing_spans: u32) -> DispatchResult { - let controller = >::get(stash).ok_or(Error::::NotStash)?; - - slashing::clear_stash_metadata::(stash, num_slashing_spans)?; - - >::remove(stash); - >::remove(&controller); - - >::remove(stash); - Self::do_remove_validator(stash); - Self::do_remove_nominator(stash); - - frame_system::Pallet::::dec_consumers(stash); - - Ok(()) - } - - /// Clear all era information for given era. - fn clear_era_information(era_index: EraIndex) { - >::remove_prefix(era_index, None); - >::remove_prefix(era_index, None); - >::remove_prefix(era_index, None); - >::remove(era_index); - >::remove(era_index); - >::remove(era_index); - ErasStartSessionIndex::::remove(era_index); - } - - /// Apply previously-unapplied slashes on the beginning of a new era, after a delay. - fn apply_unapplied_slashes(active_era: EraIndex) { - let slash_defer_duration = T::SlashDeferDuration::get(); - ::EarliestUnappliedSlash::mutate(|earliest| { - if let Some(ref mut earliest) = earliest { - let keep_from = active_era.saturating_sub(slash_defer_duration); - for era in (*earliest)..keep_from { - let era_slashes = ::UnappliedSlashes::take(&era); - for slash in era_slashes { - slashing::apply_slash::(slash); - } - } - - *earliest = (*earliest).max(keep_from) - } - }) - } - - /// Add reward points to validators using their stash account ID. - /// - /// Validators are keyed by stash account ID and must be in the current elected set. - /// - /// For each element in the iterator the given number of points in u32 is added to the - /// validator, thus duplicates are handled. - /// - /// At the end of the era each the total payout will be distributed among validator - /// relatively to their points. - /// - /// COMPLEXITY: Complexity is `number_of_validator_to_reward x current_elected_len`. - pub fn reward_by_ids(validators_points: impl IntoIterator) { - if let Some(active_era) = Self::active_era() { - >::mutate(active_era.index, |era_rewards| { - for (validator, points) in validators_points.into_iter() { - *era_rewards.individual.entry(validator).or_default() += points; - era_rewards.total += points; - } - }); - } - } - - /// Ensures that at the end of the current session there will be a new era. - fn ensure_new_era() { - match ForceEra::::get() { - Forcing::ForceAlways | Forcing::ForceNew => (), - _ => ForceEra::::put(Forcing::ForceNew), - } - } - - #[cfg(feature = "runtime-benchmarks")] - pub fn add_era_stakers( - current_era: EraIndex, - controller: T::AccountId, - exposure: Exposure>, - ) { - >::insert(¤t_era, &controller, &exposure); - } - - #[cfg(feature = "runtime-benchmarks")] - pub fn set_slash_reward_fraction(fraction: Perbill) { - SlashRewardFraction::::put(fraction); - } - - /// Get all of the voters that are eligible for the npos election. - /// - /// `voter_count` imposes an implicit cap on the number of voters returned; care should be taken - /// to ensure that it is accurate. - /// - /// This will use all on-chain nominators, and all the validators will inject a self vote. - /// - /// ### Slashing - /// - /// All nominations that have been submitted before the last non-zero slash of the validator are - /// auto-chilled. - /// - /// Note that this is fairly expensive: it must iterate over the min of `maybe_max_len` and - /// `voter_count` voters. Use with care. - pub fn get_npos_voters( - maybe_max_len: Option, - voter_count: usize, - ) -> Vec> { - let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); - - let weight_of = Self::weight_of_fn(); - // collect all slashing spans into a BTreeMap for further queries. - let slashing_spans = >::iter().collect::>(); - - VoterList::::iter() - .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) - .take(wanted_voters) - .collect() - } - - /// This is a very expensive function and result should be cached versus being called multiple times. - pub fn get_npos_targets() -> Vec { - Validators::::iter().map(|(v, _)| v).collect::>() - } - - /// This function will add a nominator to the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. - /// - /// If the nominator already exists, their nominations will be updated. - /// - /// NOTE: you must ALWAYS use this function to add a nominator to the system. Any access to - /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { - if !Nominators::::contains_key(who) { - CounterForNominators::::mutate(|x| x.saturating_inc()) - } - Nominators::::insert(who, nominations); - VoterList::::insert_as(who, VoterType::Nominator); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - } - - /// This function will remove a nominator from the `Nominators` storage map, - /// and keep track of the `CounterForNominators`. - /// - /// Returns true if `who` was removed from `Nominators`, otherwise false. - /// - /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to - /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_remove_nominator(who: &T::AccountId) -> bool { - if Nominators::::contains_key(who) { - Nominators::::remove(who); - CounterForNominators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - true - } else { - false - } - } - - /// This function will add a validator to the `Validators` storage map, and keep track of the - /// `CounterForValidators`. - /// - /// If the validator already exists, their preferences will be updated. - /// - /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to - /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { - if !Validators::::contains_key(who) { - CounterForValidators::::mutate(|x| x.saturating_inc()) - } - Validators::::insert(who, prefs); - VoterList::::insert_as(who, VoterType::Validator); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - } - - /// This function will remove a validator from the `Validators` storage map, - /// and keep track of the `CounterForValidators`. - /// - /// Returns true if `who` was removed from `Validators`, otherwise false. - /// - /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to - /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly - /// wrong. - pub fn do_remove_validator(who: &T::AccountId) -> bool { - if Validators::::contains_key(who) { - Validators::::remove(who); - CounterForValidators::::mutate(|x| x.saturating_dec()); - VoterList::::remove(who); - debug_assert_eq!(VoterList::::sanity_check(), Ok(())); - true - } else { - false - } - } - - /// Move a stash account from one bag to another, depositing an event on success. - /// - /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { - // if no voter at that node, don't do anything. - // the caller just wasted the fee to call this. - let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { - let weight_of = Self::weight_of_fn(); - VoterList::update_position_for(node, weight_of) - }); - if let Some((from, to)) = maybe_movement { - Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); - }; - maybe_movement - } -} - -impl - frame_election_provider_support::ElectionDataProvider> - for Pallet -{ - const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; - fn desired_targets() -> data_provider::Result<(u32, Weight)> { - Ok((Self::validator_count(), ::DbWeight::get().reads(1))) - } - - fn voters( - maybe_max_len: Option, - ) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec)>, Weight)> { - let nominator_count = CounterForNominators::::get(); - let validator_count = CounterForValidators::::get(); - let voter_count = nominator_count.saturating_add(validator_count) as usize; - - // check a few counters one last time... - debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); - debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); - debug_assert_eq!( - voter_count, - VoterList::::decode_len().unwrap_or_default(), - "voter_count must be accurate", - ); - - let slashing_span_count = >::iter().count(); - let weight = T::WeightInfo::get_npos_voters( - nominator_count, - validator_count, - slashing_span_count as u32, - ); - - Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) - } - - fn targets(maybe_max_len: Option) -> data_provider::Result<(Vec, Weight)> { - let target_count = CounterForValidators::::get() as usize; - - if maybe_max_len.map_or(false, |max_len| target_count > max_len) { - return Err("Target snapshot too big") - } - - let weight = ::DbWeight::get().reads(target_count as u64); - Ok((Self::get_npos_targets(), weight)) - } - - fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber { - let current_era = Self::current_era().unwrap_or(0); - let current_session = Self::current_planned_session(); - let current_era_start_session_index = - Self::eras_start_session_index(current_era).unwrap_or(0); - // Number of session in the current era or the maximum session per era if reached. - let era_progress = current_session - .saturating_sub(current_era_start_session_index) - .min(T::SessionsPerEra::get()); - - let until_this_session_end = T::NextNewSession::estimate_next_new_session(now) - .0 - .unwrap_or_default() - .saturating_sub(now); - - let session_length = T::NextNewSession::average_session_length(); - - let sessions_left: T::BlockNumber = match ForceEra::::get() { - Forcing::ForceNone => Bounded::max_value(), - Forcing::ForceNew | Forcing::ForceAlways => Zero::zero(), - Forcing::NotForcing if era_progress >= T::SessionsPerEra::get() => Zero::zero(), - Forcing::NotForcing => T::SessionsPerEra::get() - .saturating_sub(era_progress) - // One session is computed in this_session_end. - .saturating_sub(1) - .into(), - }; - - now.saturating_add( - until_this_session_end.saturating_add(sessions_left.saturating_mul(session_length)), - ) - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn add_voter(voter: T::AccountId, weight: VoteWeight, targets: Vec) { - use sp_std::convert::TryFrom; - let stake = >::try_from(weight).unwrap_or_else(|_| { - panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") - }); - >::insert(voter.clone(), voter.clone()); - >::insert( - voter.clone(), - StakingLedger { - stash: voter.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_nominator(&voter, Nominations { targets, submitted_in: 0, suppressed: false }); - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn add_target(target: T::AccountId) { - let stake = MinValidatorBond::::get() * 100u32.into(); - >::insert(target.clone(), target.clone()); - >::insert( - target.clone(), - StakingLedger { - stash: target.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_validator( - &target, - ValidatorPrefs { commission: Perbill::zero(), blocked: false }, - ); - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn clear() { - >::remove_all(None); - >::remove_all(None); - >::remove_all(None); - >::remove_all(None); - } - - #[cfg(any(feature = "runtime-benchmarks", test))] - fn put_snapshot( - voters: Vec<(T::AccountId, VoteWeight, Vec)>, - targets: Vec, - target_stake: Option, - ) { - use sp_std::convert::TryFrom; - targets.into_iter().for_each(|v| { - let stake: BalanceOf = target_stake - .and_then(|w| >::try_from(w).ok()) - .unwrap_or_else(|| MinNominatorBond::::get() * 100u32.into()); - >::insert(v.clone(), v.clone()); - >::insert( - v.clone(), - StakingLedger { - stash: v.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_validator( - &v, - ValidatorPrefs { commission: Perbill::zero(), blocked: false }, - ); - }); - - voters.into_iter().for_each(|(v, s, t)| { - let stake = >::try_from(s).unwrap_or_else(|_| { - panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") - }); - >::insert(v.clone(), v.clone()); - >::insert( - v.clone(), - StakingLedger { - stash: v.clone(), - active: stake, - total: stake, - unlocking: vec![], - claimed_rewards: vec![], - }, - ); - Self::do_add_nominator( - &v, - Nominations { targets: t, submitted_in: 0, suppressed: false }, - ); - }); - } -} - -/// In this implementation `new_session(session)` must be called before `end_session(session-1)` -/// i.e. the new session must be planned before the ending of the previous session. -/// -/// Once the first new_session is planned, all session must start and then end in order, though -/// some session can lag in between the newest session planned and the latest session started. -impl pallet_session::SessionManager for Pallet { - fn new_session(new_index: SessionIndex) -> Option> { - log!(trace, "planning new session {}", new_index); - CurrentPlannedSession::::put(new_index); - Self::new_session(new_index, false) - } - fn new_session_genesis(new_index: SessionIndex) -> Option> { - log!(trace, "planning new session {} at genesis", new_index); - CurrentPlannedSession::::put(new_index); - Self::new_session(new_index, true) - } - fn start_session(start_index: SessionIndex) { - log!(trace, "starting session {}", start_index); - Self::start_session(start_index) - } - fn end_session(end_index: SessionIndex) { - log!(trace, "ending session {}", end_index); - Self::end_session(end_index) - } -} - -impl historical::SessionManager>> - for Pallet -{ - fn new_session( - new_index: SessionIndex, - ) -> Option>)>> { - >::new_session(new_index).map(|validators| { - let current_era = Self::current_era() - // Must be some as a new era has been created. - .unwrap_or(0); - - validators - .into_iter() - .map(|v| { - let exposure = Self::eras_stakers(current_era, &v); - (v, exposure) - }) - .collect() - }) - } - fn new_session_genesis( - new_index: SessionIndex, - ) -> Option>)>> { - >::new_session_genesis(new_index).map( - |validators| { - let current_era = Self::current_era() - // Must be some as a new era has been created. - .unwrap_or(0); - - validators - .into_iter() - .map(|v| { - let exposure = Self::eras_stakers(current_era, &v); - (v, exposure) - }) - .collect() - }, - ) - } - fn start_session(start_index: SessionIndex) { - >::start_session(start_index) - } - fn end_session(end_index: SessionIndex) { - >::end_session(end_index) - } -} - -/// Add reward points to block authors: -/// * 20 points to the block producer for producing a (non-uncle) block in the relay chain, -/// * 2 points to the block producer for each reference to a previously unreferenced uncle, and -/// * 1 point to the producer of each referenced uncle block. -impl pallet_authorship::EventHandler for Pallet -where - T: Config + pallet_authorship::Config + pallet_session::Config, -{ - fn note_author(author: T::AccountId) { - Self::reward_by_ids(vec![(author, 20)]) - } - fn note_uncle(author: T::AccountId, _age: T::BlockNumber) { - Self::reward_by_ids(vec![(>::author(), 2), (author, 1)]) - } -} - /// A `Convert` implementation that finds the stash of the given controller account, /// if any. pub struct StashOf(sp_std::marker::PhantomData); @@ -3514,138 +771,6 @@ impl Convert } } -/// This is intended to be used with `FilterHistoricalOffences`. -impl - OnOffenceHandler, Weight> - for Pallet -where - T: pallet_session::Config::AccountId>, - T: pallet_session::historical::Config< - FullIdentification = Exposure<::AccountId, BalanceOf>, - FullIdentificationOf = ExposureOf, - >, - T::SessionHandler: pallet_session::SessionHandler<::AccountId>, - T::SessionManager: pallet_session::SessionManager<::AccountId>, - T::ValidatorIdOf: Convert< - ::AccountId, - Option<::AccountId>, - >, -{ - fn on_offence( - offenders: &[OffenceDetails< - T::AccountId, - pallet_session::historical::IdentificationTuple, - >], - slash_fraction: &[Perbill], - slash_session: SessionIndex, - ) -> Weight { - let reward_proportion = SlashRewardFraction::::get(); - let mut consumed_weight: Weight = 0; - let mut add_db_reads_writes = |reads, writes| { - consumed_weight += T::DbWeight::get().reads_writes(reads, writes); - }; - - let active_era = { - let active_era = Self::active_era(); - add_db_reads_writes(1, 0); - if active_era.is_none() { - // This offence need not be re-submitted. - return consumed_weight - } - active_era.expect("value checked not to be `None`; qed").index - }; - let active_era_start_session_index = Self::eras_start_session_index(active_era) - .unwrap_or_else(|| { - frame_support::print("Error: start_session_index must be set for current_era"); - 0 - }); - add_db_reads_writes(1, 0); - - let window_start = active_era.saturating_sub(T::BondingDuration::get()); - - // Fast path for active-era report - most likely. - // `slash_session` cannot be in a future active era. It must be in `active_era` or before. - let slash_era = if slash_session >= active_era_start_session_index { - active_era - } else { - let eras = BondedEras::::get(); - add_db_reads_writes(1, 0); - - // Reverse because it's more likely to find reports from recent eras. - match eras.iter().rev().filter(|&&(_, ref sesh)| sesh <= &slash_session).next() { - Some(&(ref slash_era, _)) => *slash_era, - // Before bonding period. defensive - should be filtered out. - None => return consumed_weight, - } - }; - - ::EarliestUnappliedSlash::mutate(|earliest| { - if earliest.is_none() { - *earliest = Some(active_era) - } - }); - add_db_reads_writes(1, 1); - - let slash_defer_duration = T::SlashDeferDuration::get(); - - let invulnerables = Self::invulnerables(); - add_db_reads_writes(1, 0); - - for (details, slash_fraction) in offenders.iter().zip(slash_fraction) { - let (stash, exposure) = &details.offender; - - // Skip if the validator is invulnerable. - if invulnerables.contains(stash) { - continue - } - - let unapplied = slashing::compute_slash::(slashing::SlashParams { - stash, - slash: *slash_fraction, - exposure, - slash_era, - window_start, - now: active_era, - reward_proportion, - }); - - if let Some(mut unapplied) = unapplied { - let nominators_len = unapplied.others.len() as u64; - let reporters_len = details.reporters.len() as u64; - - { - let upper_bound = 1 /* Validator/NominatorSlashInEra */ + 2 /* fetch_spans */; - let rw = upper_bound + nominators_len * upper_bound; - add_db_reads_writes(rw, rw); - } - unapplied.reporters = details.reporters.clone(); - if slash_defer_duration == 0 { - // Apply right away. - slashing::apply_slash::(unapplied); - { - let slash_cost = (6, 5); - let reward_cost = (2, 2); - add_db_reads_writes( - (1 + nominators_len) * slash_cost.0 + reward_cost.0 * reporters_len, - (1 + nominators_len) * slash_cost.1 + reward_cost.1 * reporters_len, - ); - } - } else { - // Defer to end of some `slash_defer_duration` from now. - ::UnappliedSlashes::mutate(active_era, move |for_later| { - for_later.push(unapplied) - }); - add_db_reads_writes(1, 1); - } - } else { - add_db_reads_writes(4 /* fetch_spans */, 5 /* kick_out_if_recent */) - } - } - - consumed_weight - } -} - /// Filter historical offences out and only allow those from the bonding period. pub struct FilterHistoricalOffences { _inner: sp_std::marker::PhantomData<(T, R)>, @@ -3675,8 +800,3 @@ where R::is_known_offence(offenders, time_slot) } } - -/// Check that list is sorted and has no duplicates. -fn is_sorted_and_unique(list: &[u32]) -> bool { - list.windows(2).all(|w| w[0] < w[1]) -} diff --git a/frame/staking/src/migrations.rs b/frame/staking/src/migrations.rs new file mode 100644 index 0000000000000..83f3ae9b4d83e --- /dev/null +++ b/frame/staking/src/migrations.rs @@ -0,0 +1,117 @@ +// This file is part of Substrate. + +// Copyright (C) 2020-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and + +//! Storage migrations for the Staking pallet. + +use super::*; + +pub mod v8 { + use super::{voter_bags::VoterList, *}; + use frame_support::ensure; + + pub fn pre_migrate() -> Result<(), &'static str> { + ensure!(StorageVersion::::get() == Releases::V7_0_0, "must upgrade linearly"); + ensure!(VoterList::::iter().count() == 0, "voter list already exists"); + Ok(()) + } + + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V8_0_0"); + + let migrated = VoterList::::regenerate(); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + + StorageVersion::::put(Releases::V8_0_0); + log!( + info, + "Completed staking migration to Releases::V8_0_0 with {} voters migrated", + migrated, + ); + + T::WeightInfo::regenerate( + CounterForValidators::::get(), + CounterForNominators::::get(), + ) + .saturating_add(T::DbWeight::get().reads(2)) + } +} + +pub mod v7 { + use super::*; + + pub fn pre_migrate() -> Result<(), &'static str> { + assert!(CounterForValidators::::get().is_zero(), "CounterForValidators already set."); + assert!(CounterForNominators::::get().is_zero(), "CounterForNominators already set."); + assert!(StorageVersion::::get() == Releases::V6_0_0); + Ok(()) + } + + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V7_0_0"); + let validator_count = Validators::::iter().count() as u32; + let nominator_count = Nominators::::iter().count() as u32; + + CounterForValidators::::put(validator_count); + CounterForNominators::::put(nominator_count); + + StorageVersion::::put(Releases::V7_0_0); + log!(info, "Completed staking migration to Releases::V7_0_0"); + + T::DbWeight::get().reads_writes(validator_count.saturating_add(nominator_count).into(), 2) + } +} + +pub mod v6 { + use super::*; + use frame_support::{generate_storage_alias, traits::Get, weights::Weight}; + + // NOTE: value type doesn't matter, we just set it to () here. + generate_storage_alias!(Staking, SnapshotValidators => Value<()>); + generate_storage_alias!(Staking, SnapshotNominators => Value<()>); + generate_storage_alias!(Staking, QueuedElected => Value<()>); + generate_storage_alias!(Staking, QueuedScore => Value<()>); + generate_storage_alias!(Staking, EraElectionStatus => Value<()>); + generate_storage_alias!(Staking, IsCurrentSessionFinal => Value<()>); + + /// check to execute prior to migration. + pub fn pre_migrate() -> Result<(), &'static str> { + // these may or may not exist. + log!(info, "SnapshotValidators.exits()? {:?}", SnapshotValidators::exists()); + log!(info, "SnapshotNominators.exits()? {:?}", SnapshotNominators::exists()); + log!(info, "QueuedElected.exits()? {:?}", QueuedElected::exists()); + log!(info, "QueuedScore.exits()? {:?}", QueuedScore::exists()); + // these must exist. + assert!(IsCurrentSessionFinal::exists(), "IsCurrentSessionFinal storage item not found!"); + assert!(EraElectionStatus::exists(), "EraElectionStatus storage item not found!"); + Ok(()) + } + + /// Migrate storage to v6. + pub fn migrate() -> Weight { + log!(info, "Migrating staking to Releases::V6_0_0"); + + SnapshotValidators::kill(); + SnapshotNominators::kill(); + QueuedElected::kill(); + QueuedScore::kill(); + EraElectionStatus::kill(); + IsCurrentSessionFinal::kill(); + + StorageVersion::::put(Releases::V6_0_0); + log!(info, "Done."); + T::DbWeight::get().writes(6 + 1) + } +} diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index a08b2e671042d..970b8a0f6e981 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -18,11 +18,14 @@ //! Test utilities use crate as staking; -use crate::*; +use crate::{voter_bags::VoterList, *}; use frame_election_provider_support::onchain; use frame_support::{ assert_ok, parameter_types, - traits::{Currency, FindAuthor, Get, OnInitialize, OneSessionHandler}, + traits::{ + Currency, FindAuthor, GenesisBuild, Get, Hooks, Imbalance, OnInitialize, OnUnbalanced, + OneSessionHandler, + }, weights::constants::RocksDbWeight, }; use sp_core::H256; diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs new file mode 100644 index 0000000000000..84f7317ee9091 --- /dev/null +++ b/frame/staking/src/pallet/impls.rs @@ -0,0 +1,1158 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementations for the Staking FRAME Pallet. + +use frame_election_provider_support::{data_provider, ElectionProvider, Supports, VoteWeight}; +use frame_support::{ + pallet_prelude::*, + traits::{ + Currency, CurrencyToVote, EstimateNextNewSession, Get, Imbalance, LockableCurrency, + OnUnbalanced, UnixTime, WithdrawReasons, + }, + weights::{Weight, WithPostDispatchInfo}, +}; +use frame_system::pallet_prelude::BlockNumberFor; +use pallet_session::historical; +use sp_runtime::{ + traits::{Bounded, Convert, SaturatedConversion, Saturating, Zero}, + Perbill, +}; +use sp_staking::{ + offence::{OffenceDetails, OnOffenceHandler}, + SessionIndex, +}; +use sp_std::{collections::btree_map::BTreeMap, prelude::*}; + +use crate::{ + log, slashing, + voter_bags::{self, VoterList}, + weights::WeightInfo, + ActiveEraInfo, BalanceOf, EraIndex, EraPayout, Exposure, ExposureOf, Forcing, + IndividualExposure, Nominations, PositiveImbalanceOf, RewardDestination, SessionInterface, + StakingLedger, ValidatorPrefs, VotingDataOf, +}; + +use super::{pallet::*, STAKING_ID}; + +impl Pallet { + /// The total balance that can be slashed from a stash account as of right now. + pub fn slashable_balance_of(stash: &T::AccountId) -> BalanceOf { + // Weight note: consider making the stake accessible through stash. + Self::bonded(stash).and_then(Self::ledger).map(|l| l.active).unwrap_or_default() + } + + /// Internal impl of [`Self::slashable_balance_of`] that returns [`VoteWeight`]. + pub fn slashable_balance_of_vote_weight( + stash: &T::AccountId, + issuance: BalanceOf, + ) -> VoteWeight { + T::CurrencyToVote::to_vote(Self::slashable_balance_of(stash), issuance) + } + + /// Returns a closure around `slashable_balance_of_vote_weight` that can be passed around. + /// + /// This prevents call sites from repeatedly requesting `total_issuance` from backend. But it is + /// important to be only used while the total issuance is not changing. + pub fn weight_of_fn() -> Box VoteWeight> { + // NOTE: changing this to unboxed `impl Fn(..)` return type and the pallet will still + // compile, while some types in mock fail to resolve. + let issuance = T::Currency::total_issuance(); + Box::new(move |who: &T::AccountId| -> VoteWeight { + Self::slashable_balance_of_vote_weight(who, issuance) + }) + } + + pub(super) fn do_payout_stakers( + validator_stash: T::AccountId, + era: EraIndex, + ) -> DispatchResultWithPostInfo { + // Validate input data + let current_era = CurrentEra::::get().ok_or_else(|| { + Error::::InvalidEraToReward + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + })?; + let history_depth = Self::history_depth(); + ensure!( + era <= current_era && era >= current_era.saturating_sub(history_depth), + Error::::InvalidEraToReward + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + ); + + // Note: if era has no reward to be claimed, era may be future. better not to update + // `ledger.claimed_rewards` in this case. + let era_payout = >::get(&era).ok_or_else(|| { + Error::::InvalidEraToReward + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + })?; + + let controller = Self::bonded(&validator_stash).ok_or_else(|| { + Error::::NotStash.with_weight(T::WeightInfo::payout_stakers_alive_staked(0)) + })?; + let mut ledger = >::get(&controller).ok_or(Error::::NotController)?; + + ledger + .claimed_rewards + .retain(|&x| x >= current_era.saturating_sub(history_depth)); + match ledger.claimed_rewards.binary_search(&era) { + Ok(_) => Err(Error::::AlreadyClaimed + .with_weight(T::WeightInfo::payout_stakers_alive_staked(0)))?, + Err(pos) => ledger.claimed_rewards.insert(pos, era), + } + + let exposure = >::get(&era, &ledger.stash); + + // Input data seems good, no errors allowed after this point + + >::insert(&controller, &ledger); + + // Get Era reward points. It has TOTAL and INDIVIDUAL + // Find the fraction of the era reward that belongs to the validator + // Take that fraction of the eras rewards to split to nominator and validator + // + // Then look at the validator, figure out the proportion of their reward + // which goes to them and each of their nominators. + + let era_reward_points = >::get(&era); + let total_reward_points = era_reward_points.total; + let validator_reward_points = era_reward_points + .individual + .get(&ledger.stash) + .map(|points| *points) + .unwrap_or_else(|| Zero::zero()); + + // Nothing to do if they have no reward points. + if validator_reward_points.is_zero() { + return Ok(Some(T::WeightInfo::payout_stakers_alive_staked(0)).into()) + } + + // This is the fraction of the total reward that the validator and the + // nominators will get. + let validator_total_reward_part = + Perbill::from_rational(validator_reward_points, total_reward_points); + + // This is how much validator + nominators are entitled to. + let validator_total_payout = validator_total_reward_part * era_payout; + + let validator_prefs = Self::eras_validator_prefs(&era, &validator_stash); + // Validator first gets a cut off the top. + let validator_commission = validator_prefs.commission; + let validator_commission_payout = validator_commission * validator_total_payout; + + let validator_leftover_payout = validator_total_payout - validator_commission_payout; + // Now let's calculate how this is split to the validator. + let validator_exposure_part = Perbill::from_rational(exposure.own, exposure.total); + let validator_staking_payout = validator_exposure_part * validator_leftover_payout; + + // We can now make total validator payout: + if let Some(imbalance) = + Self::make_payout(&ledger.stash, validator_staking_payout + validator_commission_payout) + { + Self::deposit_event(Event::::Reward(ledger.stash, imbalance.peek())); + } + + // Track the number of payout ops to nominators. Note: `WeightInfo::payout_stakers_alive_staked` + // always assumes at least a validator is paid out, so we do not need to count their payout op. + let mut nominator_payout_count: u32 = 0; + + // Lets now calculate how this is split to the nominators. + // Reward only the clipped exposures. Note this is not necessarily sorted. + for nominator in exposure.others.iter() { + let nominator_exposure_part = Perbill::from_rational(nominator.value, exposure.total); + + let nominator_reward: BalanceOf = + nominator_exposure_part * validator_leftover_payout; + // We can now make nominator payout: + if let Some(imbalance) = Self::make_payout(&nominator.who, nominator_reward) { + // Note: this logic does not count payouts for `RewardDestination::None`. + nominator_payout_count += 1; + Self::deposit_event(Event::::Reward(nominator.who.clone(), imbalance.peek())); + } + } + + debug_assert!(nominator_payout_count <= T::MaxNominatorRewardedPerValidator::get()); + Ok(Some(T::WeightInfo::payout_stakers_alive_staked(nominator_payout_count)).into()) + } + + /// Update the ledger for a controller. + /// + /// This will also update the stash lock. + pub(crate) fn update_ledger( + controller: &T::AccountId, + ledger: &StakingLedger>, + ) { + T::Currency::set_lock(STAKING_ID, &ledger.stash, ledger.total, WithdrawReasons::all()); + >::insert(controller, ledger); + } + + /// Chill a stash account. + pub(crate) fn chill_stash(stash: &T::AccountId) { + let chilled_as_validator = Self::do_remove_validator(stash); + let chilled_as_nominator = Self::do_remove_nominator(stash); + if chilled_as_validator || chilled_as_nominator { + Self::deposit_event(Event::::Chilled(stash.clone())); + } + } + + /// Actually make a payment to a staker. This uses the currency's reward function + /// to pay the right payee for the given staker account. + fn make_payout(stash: &T::AccountId, amount: BalanceOf) -> Option> { + let dest = Self::payee(stash); + match dest { + RewardDestination::Controller => Self::bonded(stash) + .and_then(|controller| Some(T::Currency::deposit_creating(&controller, amount))), + RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), + RewardDestination::Staked => Self::bonded(stash) + .and_then(|c| Self::ledger(&c).map(|l| (c, l))) + .and_then(|(controller, mut l)| { + l.active += amount; + l.total += amount; + let r = T::Currency::deposit_into_existing(stash, amount).ok(); + Self::update_ledger(&controller, &l); + r + }), + RewardDestination::Account(dest_account) => + Some(T::Currency::deposit_creating(&dest_account, amount)), + RewardDestination::None => None, + } + } + + /// Plan a new session potentially trigger a new era. + fn new_session(session_index: SessionIndex, is_genesis: bool) -> Option> { + if let Some(current_era) = Self::current_era() { + // Initial era has been set. + let current_era_start_session_index = Self::eras_start_session_index(current_era) + .unwrap_or_else(|| { + frame_support::print("Error: start_session_index must be set for current_era"); + 0 + }); + + let era_length = + session_index.checked_sub(current_era_start_session_index).unwrap_or(0); // Must never happen. + + match ForceEra::::get() { + // Will be set to `NotForcing` again if a new era has been triggered. + Forcing::ForceNew => (), + // Short circuit to `try_trigger_new_era`. + Forcing::ForceAlways => (), + // Only go to `try_trigger_new_era` if deadline reached. + Forcing::NotForcing if era_length >= T::SessionsPerEra::get() => (), + _ => { + // Either `Forcing::ForceNone`, + // or `Forcing::NotForcing if era_length >= T::SessionsPerEra::get()`. + return None + }, + } + + // New era. + let maybe_new_era_validators = Self::try_trigger_new_era(session_index, is_genesis); + if maybe_new_era_validators.is_some() && + matches!(ForceEra::::get(), Forcing::ForceNew) + { + ForceEra::::put(Forcing::NotForcing); + } + + maybe_new_era_validators + } else { + // Set initial era. + log!(debug, "Starting the first era."); + Self::try_trigger_new_era(session_index, is_genesis) + } + } + + /// Start a session potentially starting an era. + fn start_session(start_session: SessionIndex) { + let next_active_era = Self::active_era().map(|e| e.index + 1).unwrap_or(0); + // This is only `Some` when current era has already progressed to the next era, while the + // active era is one behind (i.e. in the *last session of the active era*, or *first session + // of the new current era*, depending on how you look at it). + if let Some(next_active_era_start_session_index) = + Self::eras_start_session_index(next_active_era) + { + if next_active_era_start_session_index == start_session { + Self::start_era(start_session); + } else if next_active_era_start_session_index < start_session { + // This arm should never happen, but better handle it than to stall the staking + // pallet. + frame_support::print("Warning: A session appears to have been skipped."); + Self::start_era(start_session); + } + } + } + + /// End a session potentially ending an era. + fn end_session(session_index: SessionIndex) { + if let Some(active_era) = Self::active_era() { + if let Some(next_active_era_start_session_index) = + Self::eras_start_session_index(active_era.index + 1) + { + if next_active_era_start_session_index == session_index + 1 { + Self::end_era(active_era, session_index); + } + } + } + } + + /// * Increment `active_era.index`, + /// * reset `active_era.start`, + /// * update `BondedEras` and apply slashes. + fn start_era(start_session: SessionIndex) { + let active_era = ActiveEra::::mutate(|active_era| { + let new_index = active_era.as_ref().map(|info| info.index + 1).unwrap_or(0); + *active_era = Some(ActiveEraInfo { + index: new_index, + // Set new active era start in next `on_finalize`. To guarantee usage of `Time` + start: None, + }); + new_index + }); + + let bonding_duration = T::BondingDuration::get(); + + BondedEras::::mutate(|bonded| { + bonded.push((active_era, start_session)); + + if active_era > bonding_duration { + let first_kept = active_era - bonding_duration; + + // Prune out everything that's from before the first-kept index. + let n_to_prune = + bonded.iter().take_while(|&&(era_idx, _)| era_idx < first_kept).count(); + + // Kill slashing metadata. + for (pruned_era, _) in bonded.drain(..n_to_prune) { + slashing::clear_era_metadata::(pruned_era); + } + + if let Some(&(_, first_session)) = bonded.first() { + T::SessionInterface::prune_historical_up_to(first_session); + } + } + }); + + Self::apply_unapplied_slashes(active_era); + } + + /// Compute payout for era. + fn end_era(active_era: ActiveEraInfo, _session_index: SessionIndex) { + // Note: active_era_start can be None if end era is called during genesis config. + if let Some(active_era_start) = active_era.start { + let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); + + let era_duration = (now_as_millis_u64 - active_era_start).saturated_into::(); + let staked = Self::eras_total_stake(&active_era.index); + let issuance = T::Currency::total_issuance(); + let (validator_payout, rest) = T::EraPayout::era_payout(staked, issuance, era_duration); + + Self::deposit_event(Event::::EraPayout(active_era.index, validator_payout, rest)); + + // Set ending era reward. + >::insert(&active_era.index, validator_payout); + T::RewardRemainder::on_unbalanced(T::Currency::issue(rest)); + } + } + + /// Plan a new era. + /// + /// * Bump the current era storage (which holds the latest planned era). + /// * Store start session index for the new planned era. + /// * Clean old era information. + /// * Store staking information for the new planned era + /// + /// Returns the new validator set. + pub fn trigger_new_era( + start_session_index: SessionIndex, + exposures: Vec<(T::AccountId, Exposure>)>, + ) -> Vec { + // Increment or set current era. + let new_planned_era = CurrentEra::::mutate(|s| { + *s = Some(s.map(|s| s + 1).unwrap_or(0)); + s.unwrap() + }); + ErasStartSessionIndex::::insert(&new_planned_era, &start_session_index); + + // Clean old era information. + if let Some(old_era) = new_planned_era.checked_sub(Self::history_depth() + 1) { + Self::clear_era_information(old_era); + } + + // Set staking information for the new era. + Self::store_stakers_info(exposures, new_planned_era) + } + + /// Potentially plan a new era. + /// + /// Get election result from `T::ElectionProvider`. + /// In case election result has more than [`MinimumValidatorCount`] validator trigger a new era. + /// + /// In case a new era is planned, the new validator set is returned. + pub(crate) fn try_trigger_new_era( + start_session_index: SessionIndex, + is_genesis: bool, + ) -> Option> { + let (election_result, weight) = if is_genesis { + T::GenesisElectionProvider::elect().map_err(|e| { + log!(warn, "genesis election provider failed due to {:?}", e); + Self::deposit_event(Event::StakingElectionFailed); + }) + } else { + T::ElectionProvider::elect().map_err(|e| { + log!(warn, "election provider failed due to {:?}", e); + Self::deposit_event(Event::StakingElectionFailed); + }) + } + .ok()?; + + >::register_extra_weight_unchecked( + weight, + frame_support::weights::DispatchClass::Mandatory, + ); + + let exposures = Self::collect_exposures(election_result); + + if (exposures.len() as u32) < Self::minimum_validator_count().max(1) { + // Session will panic if we ever return an empty validator set, thus max(1) ^^. + match CurrentEra::::get() { + Some(current_era) if current_era > 0 => log!( + warn, + "chain does not have enough staking candidates to operate for era {:?} ({} \ + elected, minimum is {})", + CurrentEra::::get().unwrap_or(0), + exposures.len(), + Self::minimum_validator_count(), + ), + None => { + // The initial era is allowed to have no exposures. + // In this case the SessionManager is expected to choose a sensible validator + // set. + // TODO: this should be simplified #8911 + CurrentEra::::put(0); + ErasStartSessionIndex::::insert(&0, &start_session_index); + }, + _ => (), + } + + Self::deposit_event(Event::StakingElectionFailed); + return None + } + + Self::deposit_event(Event::StakingElection); + Some(Self::trigger_new_era(start_session_index, exposures)) + } + + /// Process the output of the election. + /// + /// Store staking information for the new planned era + pub fn store_stakers_info( + exposures: Vec<(T::AccountId, Exposure>)>, + new_planned_era: EraIndex, + ) -> Vec { + let elected_stashes = exposures.iter().cloned().map(|(x, _)| x).collect::>(); + + // Populate stakers, exposures, and the snapshot of validator prefs. + let mut total_stake: BalanceOf = Zero::zero(); + exposures.into_iter().for_each(|(stash, exposure)| { + total_stake = total_stake.saturating_add(exposure.total); + >::insert(new_planned_era, &stash, &exposure); + + let mut exposure_clipped = exposure; + let clipped_max_len = T::MaxNominatorRewardedPerValidator::get() as usize; + if exposure_clipped.others.len() > clipped_max_len { + exposure_clipped.others.sort_by(|a, b| a.value.cmp(&b.value).reverse()); + exposure_clipped.others.truncate(clipped_max_len); + } + >::insert(&new_planned_era, &stash, exposure_clipped); + }); + + // Insert current era staking information + >::insert(&new_planned_era, total_stake); + + // Collect the pref of all winners. + for stash in &elected_stashes { + let pref = Self::validators(stash); + >::insert(&new_planned_era, stash, pref); + } + + if new_planned_era > 0 { + log!( + info, + "new validator set of size {:?} has been processed for era {:?}", + elected_stashes.len(), + new_planned_era, + ); + } + + elected_stashes + } + + /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a + /// [`Exposure`]. + fn collect_exposures( + supports: Supports, + ) -> Vec<(T::AccountId, Exposure>)> { + let total_issuance = T::Currency::total_issuance(); + let to_currency = |e: frame_election_provider_support::ExtendedBalance| { + T::CurrencyToVote::to_currency(e, total_issuance) + }; + + supports + .into_iter() + .map(|(validator, support)| { + // Build `struct exposure` from `support`. + let mut others = Vec::with_capacity(support.voters.len()); + let mut own: BalanceOf = Zero::zero(); + let mut total: BalanceOf = Zero::zero(); + support + .voters + .into_iter() + .map(|(nominator, weight)| (nominator, to_currency(weight))) + .for_each(|(nominator, stake)| { + if nominator == validator { + own = own.saturating_add(stake); + } else { + others.push(IndividualExposure { who: nominator, value: stake }); + } + total = total.saturating_add(stake); + }); + + let exposure = Exposure { own, others, total }; + (validator, exposure) + }) + .collect::)>>() + } + + /// Remove all associated data of a stash account from the staking system. + /// + /// Assumes storage is upgraded before calling. + /// + /// This is called: + /// - after a `withdraw_unbonded()` call that frees all of a stash's bonded balance. + /// - through `reap_stash()` if the balance has fallen to zero (through slashing). + pub(crate) fn kill_stash(stash: &T::AccountId, num_slashing_spans: u32) -> DispatchResult { + let controller = >::get(stash).ok_or(Error::::NotStash)?; + + slashing::clear_stash_metadata::(stash, num_slashing_spans)?; + + >::remove(stash); + >::remove(&controller); + + >::remove(stash); + Self::do_remove_validator(stash); + Self::do_remove_nominator(stash); + + frame_system::Pallet::::dec_consumers(stash); + + Ok(()) + } + + /// Clear all era information for given era. + pub(super) fn clear_era_information(era_index: EraIndex) { + >::remove_prefix(era_index, None); + >::remove_prefix(era_index, None); + >::remove_prefix(era_index, None); + >::remove(era_index); + >::remove(era_index); + >::remove(era_index); + ErasStartSessionIndex::::remove(era_index); + } + + /// Apply previously-unapplied slashes on the beginning of a new era, after a delay. + fn apply_unapplied_slashes(active_era: EraIndex) { + let slash_defer_duration = T::SlashDeferDuration::get(); + ::EarliestUnappliedSlash::mutate(|earliest| { + if let Some(ref mut earliest) = earliest { + let keep_from = active_era.saturating_sub(slash_defer_duration); + for era in (*earliest)..keep_from { + let era_slashes = ::UnappliedSlashes::take(&era); + for slash in era_slashes { + slashing::apply_slash::(slash); + } + } + + *earliest = (*earliest).max(keep_from) + } + }) + } + + /// Add reward points to validators using their stash account ID. + /// + /// Validators are keyed by stash account ID and must be in the current elected set. + /// + /// For each element in the iterator the given number of points in u32 is added to the + /// validator, thus duplicates are handled. + /// + /// At the end of the era each the total payout will be distributed among validator + /// relatively to their points. + /// + /// COMPLEXITY: Complexity is `number_of_validator_to_reward x current_elected_len`. + pub fn reward_by_ids(validators_points: impl IntoIterator) { + if let Some(active_era) = Self::active_era() { + >::mutate(active_era.index, |era_rewards| { + for (validator, points) in validators_points.into_iter() { + *era_rewards.individual.entry(validator).or_default() += points; + era_rewards.total += points; + } + }); + } + } + + /// Ensures that at the end of the current session there will be a new era. + pub(crate) fn ensure_new_era() { + match ForceEra::::get() { + Forcing::ForceAlways | Forcing::ForceNew => (), + _ => ForceEra::::put(Forcing::ForceNew), + } + } + + #[cfg(feature = "runtime-benchmarks")] + pub fn add_era_stakers( + current_era: EraIndex, + controller: T::AccountId, + exposure: Exposure>, + ) { + >::insert(¤t_era, &controller, &exposure); + } + + #[cfg(feature = "runtime-benchmarks")] + pub fn set_slash_reward_fraction(fraction: Perbill) { + SlashRewardFraction::::put(fraction); + } + + /// Get all of the voters that are eligible for the npos election. + /// + /// `voter_count` imposes an implicit cap on the number of voters returned; care should be taken + /// to ensure that it is accurate. + /// + /// This will use all on-chain nominators, and all the validators will inject a self vote. + /// + /// ### Slashing + /// + /// All nominations that have been submitted before the last non-zero slash of the validator are + /// auto-chilled. + /// + /// Note that this is fairly expensive: it must iterate over the min of `maybe_max_len` and + /// `voter_count` voters. Use with care. + pub fn get_npos_voters( + maybe_max_len: Option, + voter_count: usize, + ) -> Vec> { + let wanted_voters = maybe_max_len.unwrap_or(voter_count).min(voter_count); + + let weight_of = Self::weight_of_fn(); + // collect all slashing spans into a BTreeMap for further queries. + let slashing_spans = >::iter().collect::>(); + + VoterList::::iter() + .filter_map(|node| node.voting_data(&weight_of, &slashing_spans)) + .take(wanted_voters) + .collect() + } + + /// This is a very expensive function and result should be cached versus being called multiple times. + pub fn get_npos_targets() -> Vec { + Validators::::iter().map(|(v, _)| v).collect::>() + } + + /// This function will add a nominator to the `Nominators` storage map, + /// and keep track of the `CounterForNominators`. + /// + /// If the nominator already exists, their nominations will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a nominator to the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_add_nominator(who: &T::AccountId, nominations: Nominations) { + if !Nominators::::contains_key(who) { + CounterForNominators::::mutate(|x| x.saturating_inc()) + } + Nominators::::insert(who, nominations); + VoterList::::insert_as(who, voter_bags::VoterType::Nominator); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + } + + /// This function will remove a nominator from the `Nominators` storage map, + /// and keep track of the `CounterForNominators`. + /// + /// Returns true if `who` was removed from `Nominators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a nominator from the system. Any access to + /// `Nominators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_remove_nominator(who: &T::AccountId) -> bool { + if Nominators::::contains_key(who) { + Nominators::::remove(who); + CounterForNominators::::mutate(|x| x.saturating_dec()); + VoterList::::remove(who); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + true + } else { + false + } + } + + /// This function will add a validator to the `Validators` storage map, and keep track of the + /// `CounterForValidators`. + /// + /// If the validator already exists, their preferences will be updated. + /// + /// NOTE: you must ALWAYS use this function to add a validator to the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_add_validator(who: &T::AccountId, prefs: ValidatorPrefs) { + if !Validators::::contains_key(who) { + CounterForValidators::::mutate(|x| x.saturating_inc()) + } + Validators::::insert(who, prefs); + VoterList::::insert_as(who, voter_bags::VoterType::Validator); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + } + + /// This function will remove a validator from the `Validators` storage map, + /// and keep track of the `CounterForValidators`. + /// + /// Returns true if `who` was removed from `Validators`, otherwise false. + /// + /// NOTE: you must ALWAYS use this function to remove a validator from the system. Any access to + /// `Validators`, its counter, or `VoterList` outside of this function is almost certainly + /// wrong. + pub fn do_remove_validator(who: &T::AccountId) -> bool { + if Validators::::contains_key(who) { + Validators::::remove(who); + CounterForValidators::::mutate(|x| x.saturating_dec()); + VoterList::::remove(who); + debug_assert_eq!(VoterList::::sanity_check(), Ok(())); + true + } else { + false + } + } + + /// Move a stash account from one bag to another, depositing an event on success. + /// + /// If the stash changed bags, returns `Some((from, to))`. + pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { + // if no voter at that node, don't do anything. + // the caller just wasted the fee to call this. + let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { + let weight_of = Self::weight_of_fn(); + VoterList::update_position_for(node, weight_of) + }); + if let Some((from, to)) = maybe_movement { + Self::deposit_event(Event::::Rebagged(stash.clone(), from, to)); + }; + maybe_movement + } +} + +impl + frame_election_provider_support::ElectionDataProvider> + for Pallet +{ + const MAXIMUM_VOTES_PER_VOTER: u32 = T::MAX_NOMINATIONS; + fn desired_targets() -> data_provider::Result<(u32, Weight)> { + Ok((Self::validator_count(), ::DbWeight::get().reads(1))) + } + + fn voters( + maybe_max_len: Option, + ) -> data_provider::Result<(Vec<(T::AccountId, VoteWeight, Vec)>, Weight)> { + let nominator_count = CounterForNominators::::get(); + let validator_count = CounterForValidators::::get(); + let voter_count = nominator_count.saturating_add(validator_count) as usize; + + // check a few counters one last time... + debug_assert!(>::iter().count() as u32 == CounterForNominators::::get()); + debug_assert!(>::iter().count() as u32 == CounterForValidators::::get()); + debug_assert_eq!( + voter_count, + VoterList::::decode_len().unwrap_or_default(), + "voter_count must be accurate", + ); + + let slashing_span_count = >::iter().count(); + let weight = T::WeightInfo::get_npos_voters( + nominator_count, + validator_count, + slashing_span_count as u32, + ); + + Ok((Self::get_npos_voters(maybe_max_len, voter_count), weight)) + } + + fn targets(maybe_max_len: Option) -> data_provider::Result<(Vec, Weight)> { + let target_count = CounterForValidators::::get() as usize; + + if maybe_max_len.map_or(false, |max_len| target_count > max_len) { + return Err("Target snapshot too big") + } + + let weight = ::DbWeight::get().reads(target_count as u64); + Ok((Self::get_npos_targets(), weight)) + } + + fn next_election_prediction(now: T::BlockNumber) -> T::BlockNumber { + let current_era = Self::current_era().unwrap_or(0); + let current_session = Self::current_planned_session(); + let current_era_start_session_index = + Self::eras_start_session_index(current_era).unwrap_or(0); + // Number of session in the current era or the maximum session per era if reached. + let era_progress = current_session + .saturating_sub(current_era_start_session_index) + .min(T::SessionsPerEra::get()); + + let until_this_session_end = T::NextNewSession::estimate_next_new_session(now) + .0 + .unwrap_or_default() + .saturating_sub(now); + + let session_length = T::NextNewSession::average_session_length(); + + let sessions_left: T::BlockNumber = match ForceEra::::get() { + Forcing::ForceNone => Bounded::max_value(), + Forcing::ForceNew | Forcing::ForceAlways => Zero::zero(), + Forcing::NotForcing if era_progress >= T::SessionsPerEra::get() => Zero::zero(), + Forcing::NotForcing => T::SessionsPerEra::get() + .saturating_sub(era_progress) + // One session is computed in this_session_end. + .saturating_sub(1) + .into(), + }; + + now.saturating_add( + until_this_session_end.saturating_add(sessions_left.saturating_mul(session_length)), + ) + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn add_voter(voter: T::AccountId, weight: VoteWeight, targets: Vec) { + use sp_std::convert::TryFrom; + let stake = >::try_from(weight).unwrap_or_else(|_| { + panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") + }); + >::insert(voter.clone(), voter.clone()); + >::insert( + voter.clone(), + StakingLedger { + stash: voter.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_nominator(&voter, Nominations { targets, submitted_in: 0, suppressed: false }); + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn add_target(target: T::AccountId) { + let stake = MinValidatorBond::::get() * 100u32.into(); + >::insert(target.clone(), target.clone()); + >::insert( + target.clone(), + StakingLedger { + stash: target.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_validator( + &target, + ValidatorPrefs { commission: Perbill::zero(), blocked: false }, + ); + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn clear() { + >::remove_all(None); + >::remove_all(None); + >::remove_all(None); + >::remove_all(None); + } + + #[cfg(any(feature = "runtime-benchmarks", test))] + fn put_snapshot( + voters: Vec<(T::AccountId, VoteWeight, Vec)>, + targets: Vec, + target_stake: Option, + ) { + use sp_std::convert::TryFrom; + targets.into_iter().for_each(|v| { + let stake: BalanceOf = target_stake + .and_then(|w| >::try_from(w).ok()) + .unwrap_or_else(|| MinNominatorBond::::get() * 100u32.into()); + >::insert(v.clone(), v.clone()); + >::insert( + v.clone(), + StakingLedger { + stash: v.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_validator( + &v, + ValidatorPrefs { commission: Perbill::zero(), blocked: false }, + ); + }); + + voters.into_iter().for_each(|(v, s, t)| { + let stake = >::try_from(s).unwrap_or_else(|_| { + panic!("cannot convert a VoteWeight into BalanceOf, benchmark needs reconfiguring.") + }); + >::insert(v.clone(), v.clone()); + >::insert( + v.clone(), + StakingLedger { + stash: v.clone(), + active: stake, + total: stake, + unlocking: vec![], + claimed_rewards: vec![], + }, + ); + Self::do_add_nominator( + &v, + Nominations { targets: t, submitted_in: 0, suppressed: false }, + ); + }); + } +} + +/// In this implementation `new_session(session)` must be called before `end_session(session-1)` +/// i.e. the new session must be planned before the ending of the previous session. +/// +/// Once the first new_session is planned, all session must start and then end in order, though +/// some session can lag in between the newest session planned and the latest session started. +impl pallet_session::SessionManager for Pallet { + fn new_session(new_index: SessionIndex) -> Option> { + log!(trace, "planning new session {}", new_index); + CurrentPlannedSession::::put(new_index); + Self::new_session(new_index, false) + } + fn new_session_genesis(new_index: SessionIndex) -> Option> { + log!(trace, "planning new session {} at genesis", new_index); + CurrentPlannedSession::::put(new_index); + Self::new_session(new_index, true) + } + fn start_session(start_index: SessionIndex) { + log!(trace, "starting session {}", start_index); + Self::start_session(start_index) + } + fn end_session(end_index: SessionIndex) { + log!(trace, "ending session {}", end_index); + Self::end_session(end_index) + } +} + +impl historical::SessionManager>> + for Pallet +{ + fn new_session( + new_index: SessionIndex, + ) -> Option>)>> { + >::new_session(new_index).map(|validators| { + let current_era = Self::current_era() + // Must be some as a new era has been created. + .unwrap_or(0); + + validators + .into_iter() + .map(|v| { + let exposure = Self::eras_stakers(current_era, &v); + (v, exposure) + }) + .collect() + }) + } + fn new_session_genesis( + new_index: SessionIndex, + ) -> Option>)>> { + >::new_session_genesis(new_index).map( + |validators| { + let current_era = Self::current_era() + // Must be some as a new era has been created. + .unwrap_or(0); + + validators + .into_iter() + .map(|v| { + let exposure = Self::eras_stakers(current_era, &v); + (v, exposure) + }) + .collect() + }, + ) + } + fn start_session(start_index: SessionIndex) { + >::start_session(start_index) + } + fn end_session(end_index: SessionIndex) { + >::end_session(end_index) + } +} + +/// Add reward points to block authors: +/// * 20 points to the block producer for producing a (non-uncle) block in the relay chain, +/// * 2 points to the block producer for each reference to a previously unreferenced uncle, and +/// * 1 point to the producer of each referenced uncle block. +impl pallet_authorship::EventHandler for Pallet +where + T: Config + pallet_authorship::Config + pallet_session::Config, +{ + fn note_author(author: T::AccountId) { + Self::reward_by_ids(vec![(author, 20)]) + } + fn note_uncle(author: T::AccountId, _age: T::BlockNumber) { + Self::reward_by_ids(vec![(>::author(), 2), (author, 1)]) + } +} + +/// This is intended to be used with `FilterHistoricalOffences`. +impl + OnOffenceHandler, Weight> + for Pallet +where + T: pallet_session::Config::AccountId>, + T: pallet_session::historical::Config< + FullIdentification = Exposure<::AccountId, BalanceOf>, + FullIdentificationOf = ExposureOf, + >, + T::SessionHandler: pallet_session::SessionHandler<::AccountId>, + T::SessionManager: pallet_session::SessionManager<::AccountId>, + T::ValidatorIdOf: Convert< + ::AccountId, + Option<::AccountId>, + >, +{ + fn on_offence( + offenders: &[OffenceDetails< + T::AccountId, + pallet_session::historical::IdentificationTuple, + >], + slash_fraction: &[Perbill], + slash_session: SessionIndex, + ) -> Weight { + let reward_proportion = SlashRewardFraction::::get(); + let mut consumed_weight: Weight = 0; + let mut add_db_reads_writes = |reads, writes| { + consumed_weight += T::DbWeight::get().reads_writes(reads, writes); + }; + + let active_era = { + let active_era = Self::active_era(); + add_db_reads_writes(1, 0); + if active_era.is_none() { + // This offence need not be re-submitted. + return consumed_weight + } + active_era.expect("value checked not to be `None`; qed").index + }; + let active_era_start_session_index = Self::eras_start_session_index(active_era) + .unwrap_or_else(|| { + frame_support::print("Error: start_session_index must be set for current_era"); + 0 + }); + add_db_reads_writes(1, 0); + + let window_start = active_era.saturating_sub(T::BondingDuration::get()); + + // Fast path for active-era report - most likely. + // `slash_session` cannot be in a future active era. It must be in `active_era` or before. + let slash_era = if slash_session >= active_era_start_session_index { + active_era + } else { + let eras = BondedEras::::get(); + add_db_reads_writes(1, 0); + + // Reverse because it's more likely to find reports from recent eras. + match eras.iter().rev().filter(|&&(_, ref sesh)| sesh <= &slash_session).next() { + Some(&(ref slash_era, _)) => *slash_era, + // Before bonding period. defensive - should be filtered out. + None => return consumed_weight, + } + }; + + ::EarliestUnappliedSlash::mutate(|earliest| { + if earliest.is_none() { + *earliest = Some(active_era) + } + }); + add_db_reads_writes(1, 1); + + let slash_defer_duration = T::SlashDeferDuration::get(); + + let invulnerables = Self::invulnerables(); + add_db_reads_writes(1, 0); + + for (details, slash_fraction) in offenders.iter().zip(slash_fraction) { + let (stash, exposure) = &details.offender; + + // Skip if the validator is invulnerable. + if invulnerables.contains(stash) { + continue + } + + let unapplied = slashing::compute_slash::(slashing::SlashParams { + stash, + slash: *slash_fraction, + exposure, + slash_era, + window_start, + now: active_era, + reward_proportion, + }); + + if let Some(mut unapplied) = unapplied { + let nominators_len = unapplied.others.len() as u64; + let reporters_len = details.reporters.len() as u64; + + { + let upper_bound = 1 /* Validator/NominatorSlashInEra */ + 2 /* fetch_spans */; + let rw = upper_bound + nominators_len * upper_bound; + add_db_reads_writes(rw, rw); + } + unapplied.reporters = details.reporters.clone(); + if slash_defer_duration == 0 { + // Apply right away. + slashing::apply_slash::(unapplied); + { + let slash_cost = (6, 5); + let reward_cost = (2, 2); + add_db_reads_writes( + (1 + nominators_len) * slash_cost.0 + reward_cost.0 * reporters_len, + (1 + nominators_len) * slash_cost.1 + reward_cost.1 * reporters_len, + ); + } + } else { + // Defer to end of some `slash_defer_duration` from now. + ::UnappliedSlashes::mutate(active_era, move |for_later| { + for_later.push(unapplied) + }); + add_db_reads_writes(1, 1); + } + } else { + add_db_reads_writes(4 /* fetch_spans */, 5 /* kick_out_if_recent */) + } + } + + consumed_weight + } +} diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs new file mode 100644 index 0000000000000..8150d9f7cf2a5 --- /dev/null +++ b/frame/staking/src/pallet/mod.rs @@ -0,0 +1,1698 @@ +// This file is part of Substrate. + +// Copyright (C) 2017-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Staking FRAME Pallet. + +use frame_election_provider_support::VoteWeight; +use frame_support::{ + pallet_prelude::*, + traits::{ + Currency, CurrencyToVote, EnsureOrigin, EstimateNextNewSession, Get, LockIdentifier, + LockableCurrency, OnUnbalanced, UnixTime, + }, + weights::{ + constants::{WEIGHT_PER_MICROS, WEIGHT_PER_NANOS}, + Weight, + }, +}; +use frame_system::{ensure_root, ensure_signed, offchain::SendTransactionTypes, pallet_prelude::*}; +use sp_runtime::{ + traits::{CheckedSub, SaturatedConversion, StaticLookup, Zero}, + DispatchError, Perbill, Percent, +}; +use sp_staking::SessionIndex; +use sp_std::{convert::From, prelude::*, result}; + +mod impls; + +pub use impls::*; + +use crate::{ + log, migrations, slashing, voter_bags, weights::WeightInfo, AccountIdOf, ActiveEraInfo, + BalanceOf, EraIndex, EraPayout, EraRewardPoints, Exposure, Forcing, NegativeImbalanceOf, + Nominations, PositiveImbalanceOf, Releases, RewardDestination, SessionInterface, StakerStatus, + StakingLedger, UnappliedSlash, UnlockChunk, ValidatorPrefs, +}; + +pub const MAX_UNLOCKING_CHUNKS: usize = 32; +const STAKING_ID: LockIdentifier = *b"staking "; + +#[frame_support::pallet] +pub mod pallet { + use super::*; + + #[pallet::pallet] + #[pallet::generate_store(pub(crate) trait Store)] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config + SendTransactionTypes> { + /// The staking balance. + type Currency: LockableCurrency; + + /// Time used for computing era duration. + /// + /// It is guaranteed to start being called from the first `on_finalize`. Thus value at genesis + /// is not used. + type UnixTime: UnixTime; + + /// Convert a balance into a number used for election calculation. This must fit into a `u64` + /// but is allowed to be sensibly lossy. The `u64` is used to communicate with the + /// [`sp_npos_elections`] crate which accepts u64 numbers and does operations in 128. + /// Consequently, the backward convert is used convert the u128s from sp-elections back to a + /// [`BalanceOf`]. + type CurrencyToVote: CurrencyToVote>; + + /// Something that provides the election functionality. + type ElectionProvider: frame_election_provider_support::ElectionProvider< + Self::AccountId, + Self::BlockNumber, + // we only accept an election provider that has staking as data provider. + DataProvider = Pallet, + >; + + /// Something that provides the election functionality at genesis. + type GenesisElectionProvider: frame_election_provider_support::ElectionProvider< + Self::AccountId, + Self::BlockNumber, + DataProvider = Pallet, + >; + + /// Maximum number of nominations per nominator. + const MAX_NOMINATIONS: u32; + + /// Tokens have been minted and are unused for validator-reward. + /// See [Era payout](./index.html#era-payout). + type RewardRemainder: OnUnbalanced>; + + /// The overarching event type. + type Event: From> + IsType<::Event>; + + /// Handler for the unbalanced reduction when slashing a staker. + type Slash: OnUnbalanced>; + + /// Handler for the unbalanced increment when rewarding a staker. + type Reward: OnUnbalanced>; + + /// Number of sessions per era. + #[pallet::constant] + type SessionsPerEra: Get; + + /// Number of eras that staked funds must remain bonded for. + #[pallet::constant] + type BondingDuration: Get; + + /// Number of eras that slashes are deferred by, after computation. + /// + /// This should be less than the bonding duration. Set to 0 if slashes + /// should be applied immediately, without opportunity for intervention. + #[pallet::constant] + type SlashDeferDuration: Get; + + /// The origin which can cancel a deferred slash. Root can always do this. + type SlashCancelOrigin: EnsureOrigin; + + /// Interface for interacting with a session pallet. + type SessionInterface: self::SessionInterface; + + /// The payout for validators and the system for the current era. + /// See [Era payout](./index.html#era-payout). + type EraPayout: EraPayout>; + + /// Something that can estimate the next session change, accurately or as a best effort guess. + type NextNewSession: EstimateNextNewSession; + + /// The maximum number of nominators rewarded for each validator. + /// + /// For each validator only the `$MaxNominatorRewardedPerValidator` biggest stakers can claim + /// their reward. This used to limit the i/o cost for the nominator payout. + #[pallet::constant] + type MaxNominatorRewardedPerValidator: Get; + + /// Weight information for extrinsics in this pallet. + type WeightInfo: WeightInfo; + + /// The list of thresholds separating the various voter bags. + /// + /// Voters are separated into unsorted bags according to their vote weight. This specifies + /// the thresholds separating the bags. A voter's bag is the largest bag for which the + /// voter's weight is less than or equal to its upper threshold. + /// + /// When voters are iterated, higher bags are iterated completely before lower bags. This + /// means that iteration is _semi-sorted_: voters of higher weight tend to come before + /// voters of lower weight, but peer voters within a particular bag are sorted in insertion + /// order. + /// + /// # Expressing the constant + /// + /// This constant must be sorted in strictly increasing order. Duplicate items are not + /// permitted. + /// + /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// specified within the bag. For any two threshold lists, if one ends with + /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists + /// will behave identically. + /// + /// # Calculation + /// + /// It is recommended to generate the set of thresholds in a geometric series, such that + /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * + /// constant_ratio).max(threshold[k] + 1)` for all `k`. + /// + /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use + /// them, the `make-bags` feature must be enabled. + /// + /// # Examples + /// + /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, + /// and iteration is strictly in insertion order. + /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is equal to 2. + /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is approximately equal + /// to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will + /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. + /// + /// # Migration + /// + /// In the event that this list ever changes, a copy of the old bags list must be retained. + /// With that `VoterList::migrate` can be called, which will perform the appropriate + /// migration. + #[pallet::constant] + type VoterBagThresholds: Get<&'static [VoteWeight]>; + } + + #[pallet::extra_constants] + impl Pallet { + // TODO: rename to snake case after https://github.com/paritytech/substrate/issues/8826 fixed. + #[allow(non_snake_case)] + fn MaxNominations() -> u32 { + T::MAX_NOMINATIONS + } + } + + #[pallet::type_value] + pub(crate) fn HistoryDepthOnEmpty() -> u32 { + 84u32 + } + + /// Number of eras to keep in history. + /// + /// Information is kept for eras in `[current_era - history_depth; current_era]`. + /// + /// Must be more than the number of eras delayed by session otherwise. I.e. active era must + /// always be in history. I.e. `active_era > current_era - history_depth` must be + /// guaranteed. + #[pallet::storage] + #[pallet::getter(fn history_depth)] + pub(crate) type HistoryDepth = StorageValue<_, u32, ValueQuery, HistoryDepthOnEmpty>; + + /// The ideal number of staking participants. + #[pallet::storage] + #[pallet::getter(fn validator_count)] + pub type ValidatorCount = StorageValue<_, u32, ValueQuery>; + + /// Minimum number of staking participants before emergency conditions are imposed. + #[pallet::storage] + #[pallet::getter(fn minimum_validator_count)] + pub type MinimumValidatorCount = StorageValue<_, u32, ValueQuery>; + + /// Any validators that may never be slashed or forcibly kicked. It's a Vec since they're + /// easy to initialize and the performance hit is minimal (we expect no more than four + /// invulnerables) and restricted to testnets. + #[pallet::storage] + #[pallet::getter(fn invulnerables)] + pub type Invulnerables = StorageValue<_, Vec, ValueQuery>; + + /// Map from all locked "stash" accounts to the controller account. + #[pallet::storage] + #[pallet::getter(fn bonded)] + pub type Bonded = StorageMap<_, Twox64Concat, T::AccountId, T::AccountId>; + + /// The minimum active bond to become and maintain the role of a nominator. + #[pallet::storage] + pub type MinNominatorBond = StorageValue<_, BalanceOf, ValueQuery>; + + /// The minimum active bond to become and maintain the role of a validator. + #[pallet::storage] + pub type MinValidatorBond = StorageValue<_, BalanceOf, ValueQuery>; + + /// Map from all (unlocked) "controller" accounts to the info regarding the staking. + #[pallet::storage] + #[pallet::getter(fn ledger)] + pub type Ledger = + StorageMap<_, Blake2_128Concat, T::AccountId, StakingLedger>>; + + /// Where the reward payment should be made. Keyed by stash. + #[pallet::storage] + #[pallet::getter(fn payee)] + pub type Payee = + StorageMap<_, Twox64Concat, T::AccountId, RewardDestination, ValueQuery>; + + /// The map from (wannabe) validator stash key to the preferences of that validator. + /// + /// When updating this storage item, you must also update the `CounterForValidators`. + #[pallet::storage] + #[pallet::getter(fn validators)] + pub type Validators = + StorageMap<_, Twox64Concat, T::AccountId, ValidatorPrefs, ValueQuery>; + + /// A tracker to keep count of the number of items in the `Validators` map. + #[pallet::storage] + pub type CounterForValidators = StorageValue<_, u32, ValueQuery>; + + /// The maximum validator count before we stop allowing new validators to join. + /// + /// When this value is not set, no limits are enforced. + #[pallet::storage] + pub type MaxValidatorsCount = StorageValue<_, u32, OptionQuery>; + + /// The map from nominator stash key to the set of stash keys of all validators to nominate. + /// + /// When updating this storage item, you must also update the `CounterForNominators`. + #[pallet::storage] + #[pallet::getter(fn nominators)] + pub type Nominators = + StorageMap<_, Twox64Concat, T::AccountId, Nominations>; + + /// A tracker to keep count of the number of items in the `Nominators` map. + #[pallet::storage] + pub type CounterForNominators = StorageValue<_, u32, ValueQuery>; + + /// The maximum nominator count before we stop allowing new validators to join. + /// + /// When this value is not set, no limits are enforced. + #[pallet::storage] + pub type MaxNominatorsCount = StorageValue<_, u32, OptionQuery>; + + /// The current era index. + /// + /// This is the latest planned era, depending on how the Session pallet queues the validator + /// set, it might be active or not. + #[pallet::storage] + #[pallet::getter(fn current_era)] + pub type CurrentEra = StorageValue<_, EraIndex>; + + /// The active era information, it holds index and start. + /// + /// The active era is the era being currently rewarded. Validator set of this era must be + /// equal to [`SessionInterface::validators`]. + #[pallet::storage] + #[pallet::getter(fn active_era)] + pub type ActiveEra = StorageValue<_, ActiveEraInfo>; + + /// The session index at which the era start for the last `HISTORY_DEPTH` eras. + /// + /// Note: This tracks the starting session (i.e. session index when era start being active) + /// for the eras in `[CurrentEra - HISTORY_DEPTH, CurrentEra]`. + #[pallet::storage] + #[pallet::getter(fn eras_start_session_index)] + pub type ErasStartSessionIndex = StorageMap<_, Twox64Concat, EraIndex, SessionIndex>; + + /// Exposure of validator at era. + /// + /// This is keyed first by the era index to allow bulk deletion and then the stash account. + /// + /// Is it removed after `HISTORY_DEPTH` eras. + /// If stakers hasn't been set or has been removed then empty exposure is returned. + #[pallet::storage] + #[pallet::getter(fn eras_stakers)] + pub type ErasStakers = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + Exposure>, + ValueQuery, + >; + + /// Clipped Exposure of validator at era. + /// + /// This is similar to [`ErasStakers`] but number of nominators exposed is reduced to the + /// `T::MaxNominatorRewardedPerValidator` biggest stakers. + /// (Note: the field `total` and `own` of the exposure remains unchanged). + /// This is used to limit the i/o cost for the nominator payout. + /// + /// This is keyed fist by the era index to allow bulk deletion and then the stash account. + /// + /// Is it removed after `HISTORY_DEPTH` eras. + /// If stakers hasn't been set or has been removed then empty exposure is returned. + #[pallet::storage] + #[pallet::getter(fn eras_stakers_clipped)] + pub type ErasStakersClipped = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + Exposure>, + ValueQuery, + >; + + /// Similar to `ErasStakers`, this holds the preferences of validators. + /// + /// This is keyed first by the era index to allow bulk deletion and then the stash account. + /// + /// Is it removed after `HISTORY_DEPTH` eras. + // If prefs hasn't been set or has been removed then 0 commission is returned. + #[pallet::storage] + #[pallet::getter(fn eras_validator_prefs)] + pub type ErasValidatorPrefs = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + ValidatorPrefs, + ValueQuery, + >; + + /// The total validator era payout for the last `HISTORY_DEPTH` eras. + /// + /// Eras that haven't finished yet or has been removed doesn't have reward. + #[pallet::storage] + #[pallet::getter(fn eras_validator_reward)] + pub type ErasValidatorReward = StorageMap<_, Twox64Concat, EraIndex, BalanceOf>; + + /// Rewards for the last `HISTORY_DEPTH` eras. + /// If reward hasn't been set or has been removed then 0 reward is returned. + #[pallet::storage] + #[pallet::getter(fn eras_reward_points)] + pub type ErasRewardPoints = + StorageMap<_, Twox64Concat, EraIndex, EraRewardPoints, ValueQuery>; + + /// The total amount staked for the last `HISTORY_DEPTH` eras. + /// If total hasn't been set or has been removed then 0 stake is returned. + #[pallet::storage] + #[pallet::getter(fn eras_total_stake)] + pub type ErasTotalStake = + StorageMap<_, Twox64Concat, EraIndex, BalanceOf, ValueQuery>; + + /// Mode of era forcing. + #[pallet::storage] + #[pallet::getter(fn force_era)] + pub type ForceEra = StorageValue<_, Forcing, ValueQuery>; + + /// The percentage of the slash that is distributed to reporters. + /// + /// The rest of the slashed value is handled by the `Slash`. + #[pallet::storage] + #[pallet::getter(fn slash_reward_fraction)] + pub type SlashRewardFraction = StorageValue<_, Perbill, ValueQuery>; + + /// The amount of currency given to reporters of a slash event which was + /// canceled by extraordinary circumstances (e.g. governance). + #[pallet::storage] + #[pallet::getter(fn canceled_payout)] + pub type CanceledSlashPayout = StorageValue<_, BalanceOf, ValueQuery>; + + /// All unapplied slashes that are queued for later. + #[pallet::storage] + pub type UnappliedSlashes = StorageMap< + _, + Twox64Concat, + EraIndex, + Vec>>, + ValueQuery, + >; + + /// A mapping from still-bonded eras to the first session index of that era. + /// + /// Must contains information for eras for the range: + /// `[active_era - bounding_duration; active_era]` + #[pallet::storage] + pub(crate) type BondedEras = + StorageValue<_, Vec<(EraIndex, SessionIndex)>, ValueQuery>; + + /// All slashing events on validators, mapped by era to the highest slash proportion + /// and slash value of the era. + #[pallet::storage] + pub(crate) type ValidatorSlashInEra = StorageDoubleMap< + _, + Twox64Concat, + EraIndex, + Twox64Concat, + T::AccountId, + (Perbill, BalanceOf), + >; + + /// All slashing events on nominators, mapped by era to the highest slash value of the era. + #[pallet::storage] + pub(crate) type NominatorSlashInEra = + StorageDoubleMap<_, Twox64Concat, EraIndex, Twox64Concat, T::AccountId, BalanceOf>; + + /// Slashing spans for stash accounts. + #[pallet::storage] + pub(crate) type SlashingSpans = + StorageMap<_, Twox64Concat, T::AccountId, slashing::SlashingSpans>; + + /// Records information about the maximum slash of a stash within a slashing span, + /// as well as how much reward has been paid out. + #[pallet::storage] + pub(crate) type SpanSlash = StorageMap< + _, + Twox64Concat, + (T::AccountId, slashing::SpanIndex), + slashing::SpanRecord>, + ValueQuery, + >; + + /// The earliest era for which we have a pending, unapplied slash. + #[pallet::storage] + pub(crate) type EarliestUnappliedSlash = StorageValue<_, EraIndex>; + + /// The last planned session scheduled by the session pallet. + /// + /// This is basically in sync with the call to [`pallet_session::SessionManager::new_session`]. + #[pallet::storage] + #[pallet::getter(fn current_planned_session)] + pub type CurrentPlannedSession = StorageValue<_, SessionIndex, ValueQuery>; + + /// True if network has been upgraded to this version. + /// Storage version of the pallet. + /// + /// This is set to v7.0.0 for new networks. + #[pallet::storage] + pub(crate) type StorageVersion = StorageValue<_, Releases, ValueQuery>; + + // The next storage items collectively comprise the voter bags: a composite data structure + // designed to allow efficient iteration of the top N voters by stake, mostly. See + // `mod voter_bags` for details. + // + // In each of these items, voter bags are indexed by their upper weight threshold. + + /// How many voters are registered. + #[pallet::storage] + pub(crate) type CounterForVoters = StorageValue<_, u32, ValueQuery>; + + /// Which bag currently contains a particular voter. + /// + /// This may not be the appropriate bag for the voter's weight if they have been rewarded or + /// slashed. + #[pallet::storage] + pub(crate) type VoterBagFor = + StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; + + /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which + /// mainly exists to store head and tail pointers to the appropriate nodes. + #[pallet::storage] + pub(crate) type VoterBags = + StorageMap<_, Twox64Concat, VoteWeight, voter_bags::Bag>; + + /// Voter nodes store links forward and back within their respective bags, the stash id, and + /// whether the voter is a validator or nominator. + /// + /// There is nothing in this map directly identifying to which bag a particular node belongs. + /// However, the `Node` data structure has helpers which can provide that information. + #[pallet::storage] + pub(crate) type VoterNodes = + StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::Node>; + + // End of voter bags data. + + /// The threshold for when users can start calling `chill_other` for other validators / nominators. + /// The threshold is compared to the actual number of validators / nominators (`CountFor*`) in + /// the system compared to the configured max (`Max*Count`). + #[pallet::storage] + pub(crate) type ChillThreshold = StorageValue<_, Percent, OptionQuery>; + + #[pallet::genesis_config] + pub struct GenesisConfig { + pub history_depth: u32, + pub validator_count: u32, + pub minimum_validator_count: u32, + pub invulnerables: Vec, + pub force_era: Forcing, + pub slash_reward_fraction: Perbill, + pub canceled_payout: BalanceOf, + pub stakers: Vec<(T::AccountId, T::AccountId, BalanceOf, StakerStatus)>, + pub min_nominator_bond: BalanceOf, + pub min_validator_bond: BalanceOf, + } + + #[cfg(feature = "std")] + impl Default for GenesisConfig { + fn default() -> Self { + GenesisConfig { + history_depth: 84u32, + validator_count: Default::default(), + minimum_validator_count: Default::default(), + invulnerables: Default::default(), + force_era: Default::default(), + slash_reward_fraction: Default::default(), + canceled_payout: Default::default(), + stakers: Default::default(), + min_nominator_bond: Default::default(), + min_validator_bond: Default::default(), + } + } + } + + #[pallet::genesis_build] + impl GenesisBuild for GenesisConfig { + fn build(&self) { + HistoryDepth::::put(self.history_depth); + ValidatorCount::::put(self.validator_count); + MinimumValidatorCount::::put(self.minimum_validator_count); + Invulnerables::::put(&self.invulnerables); + ForceEra::::put(self.force_era); + CanceledSlashPayout::::put(self.canceled_payout); + SlashRewardFraction::::put(self.slash_reward_fraction); + StorageVersion::::put(Releases::V7_0_0); + MinNominatorBond::::put(self.min_nominator_bond); + MinValidatorBond::::put(self.min_validator_bond); + + let mut num_voters: u32 = 0; + for &(ref stash, ref controller, balance, ref status) in &self.stakers { + log!( + trace, + "inserting genesis staker: {:?} => {:?} => {:?}", + stash, + balance, + status + ); + assert!( + T::Currency::free_balance(&stash) >= balance, + "Stash does not have enough balance to bond." + ); + + if let Err(why) = >::bond( + T::Origin::from(Some(stash.clone()).into()), + T::Lookup::unlookup(controller.clone()), + balance, + RewardDestination::Staked, + ) { + // TODO: later on, fix all the tests that trigger these warnings, and + // make these assertions. Genesis stakers should all be correct! + log!(warn, "failed to bond staker at genesis: {:?}.", why); + continue + } + match status { + StakerStatus::Validator => { + if let Err(why) = >::validate( + T::Origin::from(Some(controller.clone()).into()), + Default::default(), + ) { + log!(warn, "failed to validate staker at genesis: {:?}.", why); + } else { + num_voters += 1; + } + }, + StakerStatus::Nominator(votes) => { + if let Err(why) = >::nominate( + T::Origin::from(Some(controller.clone()).into()), + votes.iter().map(|l| T::Lookup::unlookup(l.clone())).collect(), + ) { + log!(warn, "failed to nominate staker at genesis: {:?}.", why); + } else { + num_voters += 1; + } + }, + _ => (), + }; + } + + // all voters are inserted sanely. + assert_eq!( + CounterForVoters::::get(), + num_voters, + "not all genesis stakers were inserted into bags, something is wrong." + ); + } + } + + #[pallet::event] + #[pallet::generate_deposit(pub(crate) fn deposit_event)] + #[pallet::metadata(T::AccountId = "AccountId", BalanceOf = "Balance")] + pub enum Event { + /// The era payout has been set; the first balance is the validator-payout; the second is + /// the remainder from the maximum amount of reward. + /// \[era_index, validator_payout, remainder\] + EraPayout(EraIndex, BalanceOf, BalanceOf), + /// The staker has been rewarded by this amount. \[stash, amount\] + Reward(T::AccountId, BalanceOf), + /// One validator (and its nominators) has been slashed by the given amount. + /// \[validator, amount\] + Slash(T::AccountId, BalanceOf), + /// An old slashing report from a prior era was discarded because it could + /// not be processed. \[session_index\] + OldSlashingReportDiscarded(SessionIndex), + /// A new set of stakers was elected. + StakingElection, + /// An account has bonded this amount. \[stash, amount\] + /// + /// NOTE: This event is only emitted when funds are bonded via a dispatchable. Notably, + /// it will not be emitted for staking rewards when they are added to stake. + Bonded(T::AccountId, BalanceOf), + /// An account has unbonded this amount. \[stash, amount\] + Unbonded(T::AccountId, BalanceOf), + /// An account has called `withdraw_unbonded` and removed unbonding chunks worth `Balance` + /// from the unlocking queue. \[stash, amount\] + Withdrawn(T::AccountId, BalanceOf), + /// A nominator has been kicked from a validator. \[nominator, stash\] + Kicked(T::AccountId, T::AccountId), + /// The election failed. No new era is planned. + StakingElectionFailed, + /// An account has stopped participating as either a validator or nominator. + /// \[stash\] + Chilled(T::AccountId), + /// Moved an account from one bag to another. \[who, from, to\]. + Rebagged(T::AccountId, VoteWeight, VoteWeight), + } + + #[pallet::error] + pub enum Error { + /// Not a controller account. + NotController, + /// Not a stash account. + NotStash, + /// Stash is already bonded. + AlreadyBonded, + /// Controller is already paired. + AlreadyPaired, + /// Targets cannot be empty. + EmptyTargets, + /// Duplicate index. + DuplicateIndex, + /// Slash record index out of bounds. + InvalidSlashIndex, + /// Can not bond with value less than minimum required. + InsufficientBond, + /// Can not schedule more unlock chunks. + NoMoreChunks, + /// Can not rebond without unlocking chunks. + NoUnlockChunk, + /// Attempting to target a stash that still has funds. + FundedTarget, + /// Invalid era to reward. + InvalidEraToReward, + /// Invalid number of nominations. + InvalidNumberOfNominations, + /// Items are not sorted and unique. + NotSortedAndUnique, + /// Rewards for this era have already been claimed for this validator. + AlreadyClaimed, + /// Incorrect previous history depth input provided. + IncorrectHistoryDepth, + /// Incorrect number of slashing spans provided. + IncorrectSlashingSpans, + /// Internal state has become somehow corrupted and the operation cannot continue. + BadState, + /// Too many nomination targets supplied. + TooManyTargets, + /// A nomination target was supplied that was blocked or otherwise not a validator. + BadTarget, + /// The user has enough bond and thus cannot be chilled forcefully by an external person. + CannotChillOther, + /// There are too many nominators in the system. Governance needs to adjust the staking settings + /// to keep things safe for the runtime. + TooManyNominators, + /// There are too many validators in the system. Governance needs to adjust the staking settings + /// to keep things safe for the runtime. + TooManyValidators, + } + + #[pallet::hooks] + impl Hooks> for Pallet { + fn on_runtime_upgrade() -> Weight { + if StorageVersion::::get() == Releases::V6_0_0 { + migrations::v7::migrate::() + } else { + T::DbWeight::get().reads(1) + } + } + + #[cfg(feature = "try-runtime")] + fn pre_upgrade() -> Result<(), &'static str> { + if StorageVersion::::get() == Releases::V6_0_0 { + migrations::v7::pre_migrate::() + } else { + Ok(()) + } + } + + fn on_initialize(_now: BlockNumberFor) -> Weight { + // just return the weight of the on_finalize. + T::DbWeight::get().reads(1) + } + + fn on_finalize(_n: BlockNumberFor) { + // Set the start of the first era. + if let Some(mut active_era) = Self::active_era() { + if active_era.start.is_none() { + let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); + active_era.start = Some(now_as_millis_u64); + // This write only ever happens once, we don't include it in the weight in general + ActiveEra::::put(active_era); + } + } + // `on_finalize` weight is tracked in `on_initialize` + } + + fn integrity_test() { + sp_std::if_std! { + sp_io::TestExternalities::new_empty().execute_with(|| { + assert!( + T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, + "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", + T::SlashDeferDuration::get(), + T::BondingDuration::get(), + ); + + assert!( + T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "Voter bag thresholds must strictly increase", + ); + + assert!( + { + let existential_weight = voter_bags::existential_weight::(); + T::VoterBagThresholds::get() + .first() + .map(|&lowest_threshold| lowest_threshold >= existential_weight) + .unwrap_or(true) + }, + "Smallest bag should not be smaller than existential weight", + ); + }); + } + } + } + + #[pallet::call] + impl Pallet { + /// Take the origin account as a stash and lock up `value` of its balance. `controller` will + /// be the account that controls it. + /// + /// `value` must be more than the `minimum_balance` specified by `T::Currency`. + /// + /// The dispatch origin for this call must be _Signed_ by the stash account. + /// + /// Emits `Bonded`. + /// # + /// - Independent of the arguments. Moderate complexity. + /// - O(1). + /// - Three extra DB entries. + /// + /// NOTE: Two of the storage writes (`Self::bonded`, `Self::payee`) are _never_ cleaned + /// unless the `origin` falls below _existential deposit_ and gets removed as dust. + /// ------------------ + /// # + #[pallet::weight(T::WeightInfo::bond())] + pub fn bond( + origin: OriginFor, + controller: ::Source, + #[pallet::compact] value: BalanceOf, + payee: RewardDestination, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + + if >::contains_key(&stash) { + Err(Error::::AlreadyBonded)? + } + + let controller = T::Lookup::lookup(controller)?; + + if >::contains_key(&controller) { + Err(Error::::AlreadyPaired)? + } + + // Reject a bond which is considered to be _dust_. + if value < T::Currency::minimum_balance() { + Err(Error::::InsufficientBond)? + } + + frame_system::Pallet::::inc_consumers(&stash).map_err(|_| Error::::BadState)?; + + // You're auto-bonded forever, here. We might improve this by only bonding when + // you actually validate/nominate and remove once you unbond __everything__. + >::insert(&stash, &controller); + >::insert(&stash, payee); + + let current_era = CurrentEra::::get().unwrap_or(0); + let history_depth = Self::history_depth(); + let last_reward_era = current_era.saturating_sub(history_depth); + + let stash_balance = T::Currency::free_balance(&stash); + let value = value.min(stash_balance); + Self::deposit_event(Event::::Bonded(stash.clone(), value)); + let item = StakingLedger { + stash, + total: value, + active: value, + unlocking: vec![], + claimed_rewards: (last_reward_era..current_era).collect(), + }; + Self::update_ledger(&controller, &item); + Ok(()) + } + + /// Add some extra amount that have appeared in the stash `free_balance` into the balance up + /// for staking. + /// + /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. + /// + /// Use this if there are additional funds in your stash account that you wish to bond. + /// Unlike [`bond`](Self::bond) or [`unbond`](Self::unbond) this function does not impose any limitation + /// on the amount that can be added. + /// + /// Emits `Bonded`. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - O(1). + /// # + #[pallet::weight(T::WeightInfo::bond_extra())] + pub fn bond_extra( + origin: OriginFor, + #[pallet::compact] max_additional: BalanceOf, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + + let controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + + let stash_balance = T::Currency::free_balance(&stash); + if let Some(extra) = stash_balance.checked_sub(&ledger.total) { + let extra = extra.min(max_additional); + ledger.total += extra; + ledger.active += extra; + // Last check: the new active amount of ledger must be more than ED. + ensure!( + ledger.active >= T::Currency::minimum_balance(), + Error::::InsufficientBond + ); + + Self::deposit_event(Event::::Bonded(stash.clone(), extra)); + Self::update_ledger(&controller, &ledger); + Self::do_rebag(&stash); + } + Ok(()) + } + + /// Schedule a portion of the stash to be unlocked ready for transfer out after the bond + /// period ends. If this leaves an amount actively bonded less than + /// T::Currency::minimum_balance(), then it is increased to the full amount. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// Once the unlock period is done, you can call `withdraw_unbonded` to actually move + /// the funds out of management ready for transfer. + /// + /// No more than a limited number of unlocking chunks (see `MAX_UNLOCKING_CHUNKS`) + /// can co-exists at the same time. In that case, [`Call::withdraw_unbonded`] need + /// to be called first to remove some of the chunks (if possible). + /// + /// If a user encounters the `InsufficientBond` error when calling this extrinsic, + /// they should call `chill` first in order to free up their bonded funds. + /// + /// Emits `Unbonded`. + /// + /// See also [`Call::withdraw_unbonded`]. + #[pallet::weight(T::WeightInfo::unbond())] + pub fn unbond( + origin: OriginFor, + #[pallet::compact] value: BalanceOf, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks); + + let mut value = value.min(ledger.active); + + if !value.is_zero() { + ledger.active -= value; + + // Avoid there being a dust balance left in the staking system. + if ledger.active < T::Currency::minimum_balance() { + value += ledger.active; + ledger.active = Zero::zero(); + } + + let min_active_bond = if Nominators::::contains_key(&ledger.stash) { + MinNominatorBond::::get() + } else if Validators::::contains_key(&ledger.stash) { + MinValidatorBond::::get() + } else { + Zero::zero() + }; + + // Make sure that the user maintains enough active bond for their role. + // If a user runs into this error, they should chill first. + ensure!(ledger.active >= min_active_bond, Error::::InsufficientBond); + + // Note: in case there is no current era it is fine to bond one era more. + let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); + ledger.unlocking.push(UnlockChunk { value, era }); + Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); + Self::deposit_event(Event::::Unbonded(ledger.stash, value)); + } + Ok(()) + } + + /// Remove any unlocked chunks from the `unlocking` queue from our management. + /// + /// This essentially frees up that balance to be used by the stash account to do + /// whatever it wants. + /// + /// The dispatch origin for this call must be _Signed_ by the controller. + /// + /// Emits `Withdrawn`. + /// + /// See also [`Call::unbond`]. + /// + /// # + /// Complexity O(S) where S is the number of slashing spans to remove + /// NOTE: Weight annotation is the kill scenario, we refund otherwise. + /// # + #[pallet::weight(T::WeightInfo::withdraw_unbonded_kill(*num_slashing_spans))] + pub fn withdraw_unbonded( + origin: OriginFor, + num_slashing_spans: u32, + ) -> DispatchResultWithPostInfo { + let controller = ensure_signed(origin)?; + let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let (stash, old_total) = (ledger.stash.clone(), ledger.total); + if let Some(current_era) = Self::current_era() { + ledger = ledger.consolidate_unlocked(current_era) + } + + let post_info_weight = + if ledger.unlocking.is_empty() && ledger.active < T::Currency::minimum_balance() { + // This account must have called `unbond()` with some value that caused the active + // portion to fall below existential deposit + will have no more unlocking chunks + // left. We can now safely remove all staking-related information. + Self::kill_stash(&stash, num_slashing_spans)?; + // Remove the lock. + T::Currency::remove_lock(STAKING_ID, &stash); + // This is worst case scenario, so we use the full weight and return None + None + } else { + // This was the consequence of a partial unbond. just update the ledger and move on. + Self::update_ledger(&controller, &ledger); + + // This is only an update, so we use less overall weight. + Some(T::WeightInfo::withdraw_unbonded_update(num_slashing_spans)) + }; + + // `old_total` should never be less than the new total because + // `consolidate_unlocked` strictly subtracts balance. + if ledger.total < old_total { + // Already checked that this won't overflow by entry condition. + let value = old_total - ledger.total; + Self::deposit_event(Event::::Withdrawn(stash, value)); + } + + Ok(post_info_weight.into()) + } + + /// Declare the desire to validate for the origin controller. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + #[pallet::weight(T::WeightInfo::validate())] + pub fn validate(origin: OriginFor, prefs: ValidatorPrefs) -> DispatchResult { + let controller = ensure_signed(origin)?; + + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(ledger.active >= MinValidatorBond::::get(), Error::::InsufficientBond); + let stash = &ledger.stash; + + // Only check limits if they are not already a validator. + if !Validators::::contains_key(stash) { + // If this error is reached, we need to adjust the `MinValidatorBond` and start calling `chill_other`. + // Until then, we explicitly block new validators to protect the runtime. + if let Some(max_validators) = MaxValidatorsCount::::get() { + ensure!( + CounterForValidators::::get() < max_validators, + Error::::TooManyValidators + ); + } + } + + Self::do_remove_nominator(stash); + Self::do_add_validator(stash, prefs); + Ok(()) + } + + /// Declare the desire to nominate `targets` for the origin controller. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// # + /// - The transaction's complexity is proportional to the size of `targets` (N) + /// which is capped at CompactAssignments::LIMIT (MAX_NOMINATIONS). + /// - Both the reads and writes follow a similar pattern. + /// # + #[pallet::weight(T::WeightInfo::nominate(targets.len() as u32))] + pub fn nominate( + origin: OriginFor, + targets: Vec<::Source>, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(ledger.active >= MinNominatorBond::::get(), Error::::InsufficientBond); + let stash = &ledger.stash; + + // Only check limits if they are not already a nominator. + if !Nominators::::contains_key(stash) { + // If this error is reached, we need to adjust the `MinNominatorBond` and start calling `chill_other`. + // Until then, we explicitly block new nominators to protect the runtime. + if let Some(max_nominators) = MaxNominatorsCount::::get() { + ensure!( + CounterForNominators::::get() < max_nominators, + Error::::TooManyNominators + ); + } + } + + ensure!(!targets.is_empty(), Error::::EmptyTargets); + ensure!(targets.len() <= T::MAX_NOMINATIONS as usize, Error::::TooManyTargets); + + let old = Nominators::::get(stash).map_or_else(Vec::new, |x| x.targets); + + let targets = targets + .into_iter() + .map(|t| T::Lookup::lookup(t).map_err(DispatchError::from)) + .map(|n| { + n.and_then(|n| { + if old.contains(&n) || !Validators::::get(&n).blocked { + Ok(n) + } else { + Err(Error::::BadTarget.into()) + } + }) + }) + .collect::, _>>()?; + + let nominations = Nominations { + targets, + // Initial nominations are considered submitted at era 0. See `Nominations` doc + submitted_in: Self::current_era().unwrap_or(0), + suppressed: false, + }; + + Self::do_remove_validator(stash); + Self::do_add_nominator(stash, nominations); + Ok(()) + } + + /// Declare no desire to either validate or nominate. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - Contains one read. + /// - Writes are limited to the `origin` account key. + /// # + #[pallet::weight(T::WeightInfo::chill())] + pub fn chill(origin: OriginFor) -> DispatchResult { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + Self::chill_stash(&ledger.stash); + Ok(()) + } + + /// (Re-)set the payment target for a controller. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - Contains a limited number of reads. + /// - Writes are limited to the `origin` account key. + /// --------- + /// - Weight: O(1) + /// - DB Weight: + /// - Read: Ledger + /// - Write: Payee + /// # + #[pallet::weight(T::WeightInfo::set_payee())] + pub fn set_payee( + origin: OriginFor, + payee: RewardDestination, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let stash = &ledger.stash; + >::insert(stash, payee); + Ok(()) + } + + /// (Re-)set the controller of a stash. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the stash, not the controller. + /// + /// # + /// - Independent of the arguments. Insignificant complexity. + /// - Contains a limited number of reads. + /// - Writes are limited to the `origin` account key. + /// ---------- + /// Weight: O(1) + /// DB Weight: + /// - Read: Bonded, Ledger New Controller, Ledger Old Controller + /// - Write: Bonded, Ledger New Controller, Ledger Old Controller + /// # + #[pallet::weight(T::WeightInfo::set_controller())] + pub fn set_controller( + origin: OriginFor, + controller: ::Source, + ) -> DispatchResult { + let stash = ensure_signed(origin)?; + let old_controller = Self::bonded(&stash).ok_or(Error::::NotStash)?; + let controller = T::Lookup::lookup(controller)?; + if >::contains_key(&controller) { + Err(Error::::AlreadyPaired)? + } + if controller != old_controller { + >::insert(&stash, &controller); + if let Some(l) = >::take(&old_controller) { + >::insert(&controller, l); + } + } + Ok(()) + } + + /// Sets the ideal number of validators. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Weight: O(1) + /// Write: Validator Count + /// # + #[pallet::weight(T::WeightInfo::set_validator_count())] + pub fn set_validator_count( + origin: OriginFor, + #[pallet::compact] new: u32, + ) -> DispatchResult { + ensure_root(origin)?; + ValidatorCount::::put(new); + Ok(()) + } + + /// Increments the ideal number of validators. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Same as [`Self::set_validator_count`]. + /// # + #[pallet::weight(T::WeightInfo::set_validator_count())] + pub fn increase_validator_count( + origin: OriginFor, + #[pallet::compact] additional: u32, + ) -> DispatchResult { + ensure_root(origin)?; + ValidatorCount::::mutate(|n| *n += additional); + Ok(()) + } + + /// Scale up the ideal number of validators by a factor. + /// + /// The dispatch origin must be Root. + /// + /// # + /// Same as [`Self::set_validator_count`]. + /// # + #[pallet::weight(T::WeightInfo::set_validator_count())] + pub fn scale_validator_count(origin: OriginFor, factor: Percent) -> DispatchResult { + ensure_root(origin)?; + ValidatorCount::::mutate(|n| *n += factor * *n); + Ok(()) + } + + /// Force there to be no new eras indefinitely. + /// + /// The dispatch origin must be Root. + /// + /// # Warning + /// + /// The election process starts multiple blocks before the end of the era. + /// Thus the election process may be ongoing when this is called. In this case the + /// election will continue until the next era is triggered. + /// + /// # + /// - No arguments. + /// - Weight: O(1) + /// - Write: ForceEra + /// # + #[pallet::weight(T::WeightInfo::force_no_eras())] + pub fn force_no_eras(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + ForceEra::::put(Forcing::ForceNone); + Ok(()) + } + + /// Force there to be a new era at the end of the next session. After this, it will be + /// reset to normal (non-forced) behaviour. + /// + /// The dispatch origin must be Root. + /// + /// # Warning + /// + /// The election process starts multiple blocks before the end of the era. + /// If this is called just before a new era is triggered, the election process may not + /// have enough blocks to get a result. + /// + /// # + /// - No arguments. + /// - Weight: O(1) + /// - Write ForceEra + /// # + #[pallet::weight(T::WeightInfo::force_new_era())] + pub fn force_new_era(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + ForceEra::::put(Forcing::ForceNew); + Ok(()) + } + + /// Set the validators who cannot be slashed (if any). + /// + /// The dispatch origin must be Root. + /// + /// # + /// - O(V) + /// - Write: Invulnerables + /// # + #[pallet::weight(T::WeightInfo::set_invulnerables(invulnerables.len() as u32))] + pub fn set_invulnerables( + origin: OriginFor, + invulnerables: Vec, + ) -> DispatchResult { + ensure_root(origin)?; + >::put(invulnerables); + Ok(()) + } + + /// Force a current staker to become completely unstaked, immediately. + /// + /// The dispatch origin must be Root. + /// + /// # + /// O(S) where S is the number of slashing spans to be removed + /// Reads: Bonded, Slashing Spans, Account, Locks + /// Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Account, Locks + /// Writes Each: SpanSlash * S + /// # + #[pallet::weight(T::WeightInfo::force_unstake(*num_slashing_spans))] + pub fn force_unstake( + origin: OriginFor, + stash: T::AccountId, + num_slashing_spans: u32, + ) -> DispatchResult { + ensure_root(origin)?; + + // Remove all staking-related information. + Self::kill_stash(&stash, num_slashing_spans)?; + + // Remove the lock. + T::Currency::remove_lock(STAKING_ID, &stash); + Ok(()) + } + + /// Force there to be a new era at the end of sessions indefinitely. + /// + /// The dispatch origin must be Root. + /// + /// # Warning + /// + /// The election process starts multiple blocks before the end of the era. + /// If this is called just before a new era is triggered, the election process may not + /// have enough blocks to get a result. + /// + /// # + /// - Weight: O(1) + /// - Write: ForceEra + /// # + #[pallet::weight(T::WeightInfo::force_new_era_always())] + pub fn force_new_era_always(origin: OriginFor) -> DispatchResult { + ensure_root(origin)?; + ForceEra::::put(Forcing::ForceAlways); + Ok(()) + } + + /// Cancel enactment of a deferred slash. + /// + /// Can be called by the `T::SlashCancelOrigin`. + /// + /// Parameters: era and indices of the slashes for that era to kill. + /// + /// # + /// Complexity: O(U + S) + /// with U unapplied slashes weighted with U=1000 + /// and S is the number of slash indices to be canceled. + /// - Read: Unapplied Slashes + /// - Write: Unapplied Slashes + /// # + #[pallet::weight(T::WeightInfo::cancel_deferred_slash(slash_indices.len() as u32))] + pub fn cancel_deferred_slash( + origin: OriginFor, + era: EraIndex, + slash_indices: Vec, + ) -> DispatchResult { + T::SlashCancelOrigin::ensure_origin(origin)?; + + ensure!(!slash_indices.is_empty(), Error::::EmptyTargets); + ensure!(is_sorted_and_unique(&slash_indices), Error::::NotSortedAndUnique); + + let mut unapplied = ::UnappliedSlashes::get(&era); + let last_item = slash_indices[slash_indices.len() - 1]; + ensure!((last_item as usize) < unapplied.len(), Error::::InvalidSlashIndex); + + for (removed, index) in slash_indices.into_iter().enumerate() { + let index = (index as usize) - removed; + unapplied.remove(index); + } + + ::UnappliedSlashes::insert(&era, &unapplied); + Ok(()) + } + + /// Pay out all the stakers behind a single validator for a single era. + /// + /// - `validator_stash` is the stash account of the validator. Their nominators, up to + /// `T::MaxNominatorRewardedPerValidator`, will also receive their rewards. + /// - `era` may be any era between `[current_era - history_depth; current_era]`. + /// + /// The origin of this call must be _Signed_. Any account can call this function, even if + /// it is not one of the stakers. + /// + /// # + /// - Time complexity: at most O(MaxNominatorRewardedPerValidator). + /// - Contains a limited number of reads and writes. + /// ----------- + /// N is the Number of payouts for the validator (including the validator) + /// Weight: + /// - Reward Destination Staked: O(N) + /// - Reward Destination Controller (Creating): O(N) + /// + /// NOTE: weights are assuming that payouts are made to alive stash account (Staked). + /// Paying even a dead controller is cheaper weight-wise. We don't do any refunds here. + /// # + #[pallet::weight(T::WeightInfo::payout_stakers_alive_staked( + T::MaxNominatorRewardedPerValidator::get() + ))] + pub fn payout_stakers( + origin: OriginFor, + validator_stash: T::AccountId, + era: EraIndex, + ) -> DispatchResultWithPostInfo { + ensure_signed(origin)?; + Self::do_payout_stakers(validator_stash, era) + } + + /// Rebond a portion of the stash scheduled to be unlocked. + /// + /// The dispatch origin must be signed by the controller. + /// + /// # + /// - Time complexity: O(L), where L is unlocking chunks + /// - Bounded by `MAX_UNLOCKING_CHUNKS`. + /// - Storage changes: Can't increase storage, only decrease it. + /// # + #[pallet::weight(T::WeightInfo::rebond(MAX_UNLOCKING_CHUNKS as u32))] + pub fn rebond( + origin: OriginFor, + #[pallet::compact] value: BalanceOf, + ) -> DispatchResultWithPostInfo { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + ensure!(!ledger.unlocking.is_empty(), Error::::NoUnlockChunk); + + let ledger = ledger.rebond(value); + // Last check: the new active amount of ledger must be more than ED. + ensure!(ledger.active >= T::Currency::minimum_balance(), Error::::InsufficientBond); + + Self::deposit_event(Event::::Bonded(ledger.stash.clone(), value)); + Self::update_ledger(&controller, &ledger); + Self::do_rebag(&ledger.stash); + Ok(Some( + 35 * WEIGHT_PER_MICROS + + 50 * WEIGHT_PER_NANOS * (ledger.unlocking.len() as Weight) + + T::DbWeight::get().reads_writes(3, 2), + ) + .into()) + } + + /// Set `HistoryDepth` value. This function will delete any history information + /// when `HistoryDepth` is reduced. + /// + /// Parameters: + /// - `new_history_depth`: The new history depth you would like to set. + /// - `era_items_deleted`: The number of items that will be deleted by this dispatch. + /// This should report all the storage items that will be deleted by clearing old + /// era history. Needed to report an accurate weight for the dispatch. Trusted by + /// `Root` to report an accurate number. + /// + /// Origin must be root. + /// + /// # + /// - E: Number of history depths removed, i.e. 10 -> 7 = 3 + /// - Weight: O(E) + /// - DB Weight: + /// - Reads: Current Era, History Depth + /// - Writes: History Depth + /// - Clear Prefix Each: Era Stakers, EraStakersClipped, ErasValidatorPrefs + /// - Writes Each: ErasValidatorReward, ErasRewardPoints, ErasTotalStake, ErasStartSessionIndex + /// # + #[pallet::weight(T::WeightInfo::set_history_depth(*_era_items_deleted))] + pub fn set_history_depth( + origin: OriginFor, + #[pallet::compact] new_history_depth: EraIndex, + #[pallet::compact] _era_items_deleted: u32, + ) -> DispatchResult { + ensure_root(origin)?; + if let Some(current_era) = Self::current_era() { + HistoryDepth::::mutate(|history_depth| { + let last_kept = current_era.checked_sub(*history_depth).unwrap_or(0); + let new_last_kept = current_era.checked_sub(new_history_depth).unwrap_or(0); + for era_index in last_kept..new_last_kept { + Self::clear_era_information(era_index); + } + *history_depth = new_history_depth + }) + } + Ok(()) + } + + /// Remove all data structure concerning a staker/stash once its balance is at the minimum. + /// This is essentially equivalent to `withdraw_unbonded` except it can be called by anyone + /// and the target `stash` must have no funds left beyond the ED. + /// + /// This can be called from any origin. + /// + /// - `stash`: The stash account to reap. Its balance must be zero. + /// + /// # + /// Complexity: O(S) where S is the number of slashing spans on the account. + /// DB Weight: + /// - Reads: Stash Account, Bonded, Slashing Spans, Locks + /// - Writes: Bonded, Slashing Spans (if S > 0), Ledger, Payee, Validators, Nominators, Stash Account, Locks + /// - Writes Each: SpanSlash * S + /// # + #[pallet::weight(T::WeightInfo::reap_stash(*num_slashing_spans))] + pub fn reap_stash( + _origin: OriginFor, + stash: T::AccountId, + num_slashing_spans: u32, + ) -> DispatchResult { + let at_minimum = T::Currency::total_balance(&stash) == T::Currency::minimum_balance(); + ensure!(at_minimum, Error::::FundedTarget); + Self::kill_stash(&stash, num_slashing_spans)?; + T::Currency::remove_lock(STAKING_ID, &stash); + Ok(()) + } + + /// Remove the given nominations from the calling validator. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_ by the controller, not the stash. + /// + /// - `who`: A list of nominator stash accounts who are nominating this validator which + /// should no longer be nominating this validator. + /// + /// Note: Making this call only makes sense if you first set the validator preferences to + /// block any further nominations. + #[pallet::weight(T::WeightInfo::kick(who.len() as u32))] + pub fn kick( + origin: OriginFor, + who: Vec<::Source>, + ) -> DispatchResult { + let controller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let stash = &ledger.stash; + + for nom_stash in who + .into_iter() + .map(T::Lookup::lookup) + .collect::, _>>()? + .into_iter() + { + Nominators::::mutate(&nom_stash, |maybe_nom| { + if let Some(ref mut nom) = maybe_nom { + if let Some(pos) = nom.targets.iter().position(|v| v == stash) { + nom.targets.swap_remove(pos); + Self::deposit_event(Event::::Kicked( + nom_stash.clone(), + stash.clone(), + )); + } + } + }); + } + + Ok(()) + } + + /// Update the various staking limits this pallet. + /// + /// * `min_nominator_bond`: The minimum active bond needed to be a nominator. + /// * `min_validator_bond`: The minimum active bond needed to be a validator. + /// * `max_nominator_count`: The max number of users who can be a nominator at once. + /// When set to `None`, no limit is enforced. + /// * `max_validator_count`: The max number of users who can be a validator at once. + /// When set to `None`, no limit is enforced. + /// + /// Origin must be Root to call this function. + /// + /// NOTE: Existing nominators and validators will not be affected by this update. + /// to kick people under the new limits, `chill_other` should be called. + #[pallet::weight(T::WeightInfo::set_staking_limits())] + pub fn set_staking_limits( + origin: OriginFor, + min_nominator_bond: BalanceOf, + min_validator_bond: BalanceOf, + max_nominator_count: Option, + max_validator_count: Option, + threshold: Option, + ) -> DispatchResult { + ensure_root(origin)?; + MinNominatorBond::::set(min_nominator_bond); + MinValidatorBond::::set(min_validator_bond); + MaxNominatorsCount::::set(max_nominator_count); + MaxValidatorsCount::::set(max_validator_count); + ChillThreshold::::set(threshold); + Ok(()) + } + + /// Declare a `controller` to stop participating as either a validator or nominator. + /// + /// Effects will be felt at the beginning of the next era. + /// + /// The dispatch origin for this call must be _Signed_, but can be called by anyone. + /// + /// If the caller is the same as the controller being targeted, then no further checks are + /// enforced, and this function behaves just like `chill`. + /// + /// If the caller is different than the controller being targeted, the following conditions + /// must be met: + /// * A `ChillThreshold` must be set and checked which defines how close to the max + /// nominators or validators we must reach before users can start chilling one-another. + /// * A `MaxNominatorCount` and `MaxValidatorCount` must be set which is used to determine + /// how close we are to the threshold. + /// * A `MinNominatorBond` and `MinValidatorBond` must be set and checked, which determines + /// if this is a person that should be chilled because they have not met the threshold + /// bond required. + /// + /// This can be helpful if bond requirements are updated, and we need to remove old users + /// who do not satisfy these requirements. + // TODO: Maybe we can deprecate `chill` in the future. + // https://github.com/paritytech/substrate/issues/9111 + #[pallet::weight(T::WeightInfo::chill_other())] + pub fn chill_other(origin: OriginFor, controller: T::AccountId) -> DispatchResult { + // Anyone can call this function. + let caller = ensure_signed(origin)?; + let ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; + let stash = ledger.stash; + + // In order for one user to chill another user, the following conditions must be met: + // * A `ChillThreshold` is set which defines how close to the max nominators or + // validators we must reach before users can start chilling one-another. + // * A `MaxNominatorCount` and `MaxValidatorCount` which is used to determine how close + // we are to the threshold. + // * A `MinNominatorBond` and `MinValidatorBond` which is the final condition checked to + // determine this is a person that should be chilled because they have not met the + // threshold bond required. + // + // Otherwise, if caller is the same as the controller, this is just like `chill`. + if caller != controller { + let threshold = ChillThreshold::::get().ok_or(Error::::CannotChillOther)?; + let min_active_bond = if Nominators::::contains_key(&stash) { + let max_nominator_count = + MaxNominatorsCount::::get().ok_or(Error::::CannotChillOther)?; + let current_nominator_count = CounterForNominators::::get(); + ensure!( + threshold * max_nominator_count < current_nominator_count, + Error::::CannotChillOther + ); + MinNominatorBond::::get() + } else if Validators::::contains_key(&stash) { + let max_validator_count = + MaxValidatorsCount::::get().ok_or(Error::::CannotChillOther)?; + let current_validator_count = CounterForValidators::::get(); + ensure!( + threshold * max_validator_count < current_validator_count, + Error::::CannotChillOther + ); + MinValidatorBond::::get() + } else { + Zero::zero() + }; + + ensure!(ledger.active < min_active_bond, Error::::CannotChillOther); + } + + Self::chill_stash(&stash); + Ok(()) + } + + /// Declare that some `stash` has, through rewards or penalties, sufficiently changed its + /// stake that it should properly fall into a different bag than its current position. + /// + /// This will adjust its position into the appropriate bag. This will affect its position + /// among the nominator/validator set once the snapshot is prepared for the election. + /// + /// Anyone can call this function about any stash. + #[pallet::weight(T::WeightInfo::rebag())] + pub fn rebag(origin: OriginFor, stash: AccountIdOf) -> DispatchResult { + ensure_signed(origin)?; + Pallet::::do_rebag(&stash); + Ok(()) + } + } +} + +/// Check that list is sorted and has no duplicates. +fn is_sorted_and_unique(list: &[u32]) -> bool { + list.windows(2).all(|w| w[0] < w[1]) +} diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index 227043b656eef..332c9ffc39069 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -49,7 +49,7 @@ //! //! Based on research at -use super::{ +use crate::{ BalanceOf, Config, EraIndex, Error, Exposure, NegativeImbalanceOf, Pallet, Perbill, SessionInterface, Store, UnappliedSlash, }; diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 0947a1160febf..44bd84b9a167f 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -27,6 +27,11 @@ use rand_chacha::{ }; use sp_io::hashing::blake2_256; +use crate::voter_bags::VoterList; +use frame_support::{pallet_prelude::*, traits::Currency}; +use sp_runtime::{traits::StaticLookup, Perbill}; +use sp_std::prelude::*; + const SEED: u32 = 0; /// This function removes all validators and nominators from storage. diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 11432e6d68e3d..224c98490106b 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -18,10 +18,13 @@ //! Tests for the module. use super::{Event, *}; -use frame_election_provider_support::Support; +use crate::voter_bags::VoterList; +use frame_election_provider_support::{ElectionProvider, Support}; use frame_support::{ assert_noop, assert_ok, - traits::{Currency, OnInitialize, ReservableCurrency}, + dispatch::WithPostDispatchInfo, + pallet_prelude::*, + traits::{Currency, Get, OnInitialize, ReservableCurrency}, weights::{extract_actual_weight, GetDispatchInfo}, }; use mock::*; @@ -30,8 +33,13 @@ use sp_npos_elections::supports_eq_unordered; use sp_runtime::{ assert_eq_error_rate, traits::{BadOrigin, Dispatchable}, + Perbill, Percent, }; -use sp_staking::offence::OffenceDetails; +use sp_staking::{ + offence::{OffenceDetails, OnOffenceHandler}, + SessionIndex, +}; +use sp_std::prelude::*; use substrate_test_utils::assert_eq_uvec; #[test] @@ -3885,7 +3893,7 @@ mod voter_bags { // decrease stake within the range of the current bag assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 - // does not change bags + // does not change bags assert_eq!( get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] @@ -3893,7 +3901,7 @@ mod voter_bags { // reduce stake to the level of a non-existent bag assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 - // creates the bag and moves the voter into it + // creates the bag and moves the voter into it assert_eq!( get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] @@ -3901,7 +3909,7 @@ mod voter_bags { // increase stake by `rebond`-ing to the level of a pre-existing bag assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 - // moves the voter to that bag + // moves the voter to that bag assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); // TODO test rebag directly diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 04d1943164bfb..90c257fea758b 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -1464,12 +1464,7 @@ mod bags { node(voter_11, Some(101), None, bag_1000.bag_upper) ); - assert_eq!( - bag_1000, - Bag { - head: Some(11), tail: Some(11), bag_upper: 1_000 - } - ) + assert_eq!(bag_1000, Bag { head: Some(11), tail: Some(11), bag_upper: 1_000 }) }); } From 530e4d2f3d7ce39fa8074c4560911fe7d3e1377e Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Wed, 28 Jul 2021 11:11:17 -0700 Subject: [PATCH 80/82] Reduce diff --- frame/staking/src/pallet/impls.rs | 2 +- frame/staking/src/pallet/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 84f7317ee9091..4c6db870d843c 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -560,7 +560,7 @@ impl Pallet { } /// Clear all era information for given era. - pub(super) fn clear_era_information(era_index: EraIndex) { + pub(crate) fn clear_era_information(era_index: EraIndex) { >::remove_prefix(era_index, None); >::remove_prefix(era_index, None); >::remove_prefix(era_index, None); diff --git a/frame/staking/src/pallet/mod.rs b/frame/staking/src/pallet/mod.rs index 8150d9f7cf2a5..417005e155a81 100644 --- a/frame/staking/src/pallet/mod.rs +++ b/frame/staking/src/pallet/mod.rs @@ -127,7 +127,7 @@ pub mod pallet { type SlashCancelOrigin: EnsureOrigin; /// Interface for interacting with a session pallet. - type SessionInterface: self::SessionInterface; + type SessionInterface: SessionInterface; /// The payout for validators and the system for the current era. /// See [Era payout](./index.html#era-payout). @@ -932,7 +932,7 @@ pub mod pallet { ) -> DispatchResult { let controller = ensure_signed(origin)?; let mut ledger = Self::ledger(&controller).ok_or(Error::::NotController)?; - ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks); + ensure!(ledger.unlocking.len() < MAX_UNLOCKING_CHUNKS, Error::::NoMoreChunks,); let mut value = value.min(ledger.active); From df990d3d544a2d6d5da4992b8bfc8cddf5336947 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Sun, 1 Aug 2021 12:54:47 -0700 Subject: [PATCH 81/82] Add comment for test to add --- frame/staking/src/tests.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index bf07858cc4057..9c96caadd7026 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3893,7 +3893,7 @@ mod voter_bags { // decrease stake within the range of the current bag assert_ok!(Staking::unbond(Origin::signed(43), 999)); // 2000 - 999 = 1001 - // does not change bags + // does not change bags assert_eq!( get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101]), (2000, vec![42])] @@ -3901,7 +3901,7 @@ mod voter_bags { // reduce stake to the level of a non-existent bag assert_ok!(Staking::unbond(Origin::signed(43), 971)); // 1001 - 971 = 30 - // creates the bag and moves the voter into it + // creates the bag and moves the voter into it assert_eq!( get_bags(), vec![(10, vec![31]), (30, vec![42]), (1000, vec![11, 21, 101]),] @@ -3909,12 +3909,17 @@ mod voter_bags { // increase stake by `rebond`-ing to the level of a pre-existing bag assert_ok!(Staking::rebond(Origin::signed(43), 31)); // 30 + 41 = 61 - // moves the voter to that bag + // moves the voter to that bag assert_eq!(get_bags(), vec![(10, vec![31]), (1000, vec![11, 21, 101, 42]),]); // TODO test rebag directly }); } + + // #[test] TODO + // fn rebag_head_works() { + // // rebagging the head of a bag results in the old bag having a new head and an overall correct state. + // } } mod election_data_provider { From 1632747000a8ce27e023dcc1620f9735af7230e2 Mon Sep 17 00:00:00 2001 From: emostov <32168567+emostov@users.noreply.github.com> Date: Sun, 1 Aug 2021 13:15:41 -0700 Subject: [PATCH 82/82] Add in code TODO for update_position efficiency updates --- frame/staking/src/voter_bags.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs index 90c257fea758b..45e94db1be83b 100644 --- a/frame/staking/src/voter_bags.rs +++ b/frame/staking/src/voter_bags.rs @@ -243,6 +243,10 @@ impl VoterList { node.is_misplaced(&weight_of).then(move || { let old_idx = node.bag_upper; + // TODO: there should be a way to move a non-head-tail node to another bag + // with just 1 bag read of the destination bag and zero writes + // https://github.com/paritytech/substrate/pull/9468/files/83289aa4a15d61e6cb334f9d7e7f6804cb7e3537..44875c511ebdc79270100720320c8e3d2d56eb4a#r680559166 + // clear the old bag head/tail pointers as necessary if let Some(mut bag) = Bag::::get(node.bag_upper) { bag.remove_node(&node);