diff --git a/Cargo.lock b/Cargo.lock index cb45b18013992..7fc5552cdbe35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2261,6 +2261,19 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "git2" +version = "0.13.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9831e983241f8c5591ed53f17d874833e2fa82cac2625f3888c50cbfe136cba" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url 2.2.1", +] + [[package]] name = "glob" version = "0.3.0" @@ -3140,6 +3153,18 @@ version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" +[[package]] +name = "libgit2-sys" +version = "0.12.21+1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86271bacd72b2b9e854c3dcfb82efd538f15f870e4c11af66900effb462f6825" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libloading" version = "0.5.2" @@ -3635,6 +3660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" dependencies = [ "cc", + "libc", "pkg-config", "vcpkg", ] @@ -4435,6 +4461,16 @@ dependencies = [ "substrate-wasm-builder", ] +[[package]] +name = "node-runtime-voter-bags" +version = "3.0.0" +dependencies = [ + "node-runtime", + "pallet-staking", + "sp-io", + "structopt", +] + [[package]] name = "node-template" version = "3.0.0" @@ -4595,6 +4631,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-format" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" +dependencies = [ + "arrayvec 0.4.12", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -5482,12 +5528,15 @@ dependencies = [ name = "pallet-staking" version = "3.0.0" dependencies = [ + "chrono", "frame-benchmarking", "frame-election-provider-support", "frame-support", "frame-system", + "git2", "hex", "log", + "num-format", "pallet-authorship", "pallet-balances", "pallet-session", @@ -6725,14 +6774,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -6747,9 +6795,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "region" @@ -10030,18 +10078,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "fa6f76457f59514c7eeb4e59d891395fab0b2fd1d40723ae737d64153392e9c6" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "8a36768c0fbf1bb15eca10defa29526bda730a2376c2ab4393ccfa16fb1a318d" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index f7552f0bbbc48..fb25a9f120784 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "bin/node/rpc", "bin/node/rpc-client", "bin/node/runtime", + "bin/node/runtime/voter-bags", "bin/node/testing", "bin/utils/chain-spec-builder", "bin/utils/subkey", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index fd7fd4213366f..c4c394175d446 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -93,6 +93,9 @@ pub mod constants; use constants::{time::*, currency::*}; use sp_runtime::generic::Era; +/// Generated voter bag information. +mod voter_bags; + // Make the WASM binary available. #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); @@ -481,6 +484,7 @@ parameter_types! { pub const RewardCurve: &'static PiecewiseLinear<'static> = &REWARD_CURVE; pub const MaxNominatorRewardedPerValidator: u32 = 256; pub OffchainRepeat: BlockNumber = 5; + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; } use frame_election_provider_support::onchain; @@ -510,6 +514,7 @@ impl pallet_staking::Config for Runtime { type GenesisElectionProvider = onchain::OnChainSequentialPhragmen>; type WeightInfo = pallet_staking::weights::SubstrateWeight; + type VoterBagThresholds = VoterBagThresholds; } parameter_types! { diff --git a/bin/node/runtime/src/voter_bags.rs b/bin/node/runtime/src/voter_bags.rs new file mode 100644 index 0000000000000..f8f6201e94dc3 --- /dev/null +++ b/bin/node/runtime/src/voter_bags.rs @@ -0,0 +1,235 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated voter bag thresholds. +//! +//! Generated on 2021-07-05T09:17:40.469754927+00:00 +//! for the node runtime. + +/// Existential weight for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const EXISTENTIAL_WEIGHT: u64 = 100_000_000_000_000; + +/// Constant ratio between bags for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 1.0628253590743408; + +/// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 200] = [ + 100_000_000_000_000, + 106_282_535_907_434, + 112_959_774_389_150, + 120_056_512_776_105, + 127_599_106_300_477, + 135_615_565_971_369, + 144_135_662_599_590, + 153_191_037_357_827, + 162_815_319_286_803, + 173_044_250_183_800, + 183_915_817_337_347, + 195_470_394_601_017, + 207_750_892_330_229, + 220_802_916_738_890, + 234_674_939_267_673, + 249_418_476_592_914, + 265_088_281_944_639, + 281_742_548_444_211, + 299_443_125_216_738, + 318_255_747_080_822, + 338_250_278_668_647, + 359_500_973_883_001, + 382_086_751_654_776, + 406_091_489_025_036, + 431_604_332_640_068, + 458_720_029_816_222, + 487_539_280_404_019, + 518_169_110_758_247, + 550_723_271_202_866, + 585_322_658_466_782, + 622_095_764_659_305, + 661_179_154_452_653, + 702_717_972_243_610, + 746_866_481_177_808, + 793_788_636_038_393, + 843_658_692_126_636, + 896_661_852_395_681, + 952_994_955_240_703, + 1_012_867_205_499_736, + 1_076_500_951_379_881, + 1_144_132_510_194_192, + 1_216_013_045_975_769, + 1_292_409_502_228_280, + 1_373_605_593_276_862, + 1_459_902_857_901_004, + 1_551_621_779_162_291, + 1_649_102_974_585_730, + 1_752_708_461_114_642, + 1_862_822_999_536_805, + 1_979_855_523_374_646, + 2_104_240_657_545_975, + 2_236_440_332_435_128, + 2_376_945_499_368_703, + 2_526_277_953_866_680, + 2_684_992_273_439_945, + 2_853_677_877_130_641, + 3_032_961_214_443_876, + 3_223_508_091_799_862, + 3_426_026_145_146_232, + 3_641_267_467_913_124, + 3_870_031_404_070_482, + 4_113_167_516_660_186, + 4_371_578_742_827_277, + 4_646_224_747_067_156, + 4_938_125_485_141_739, + 5_248_364_991_899_922, + 5_578_095_407_069_235, + 5_928_541_253_969_291, + 6_301_003_987_036_955, + 6_696_866_825_051_405, + 7_117_599_888_008_300, + 7_564_765_656_719_910, + 8_040_024_775_416_580, + 8_545_142_218_898_723, + 9_081_993_847_142_344, + 9_652_573_371_700_016, + 10_258_999_759_768_490, + 10_903_525_103_419_522, + 11_588_542_983_217_942, + 12_316_597_357_287_042, + 13_090_392_008_832_678, + 13_912_800_587_211_472, + 14_786_877_279_832_732, + 15_715_868_154_526_436, + 16_703_223_214_499_558, + 17_752_609_210_649_358, + 18_867_923_258_814_856, + 20_053_307_312_537_008, + 21_313_163_545_075_252, + 22_652_170_697_804_756, + 24_075_301_455_707_600, + 25_587_840_914_485_432, + 27_195_406_207_875_088, + 28_903_967_368_057_400, + 30_719_869_496_628_636, + 32_649_856_328_471_220, + 34_701_095_276_033_064, + 36_881_204_047_022_752, + 39_198_278_934_370_992, + 41_660_924_883_519_016, + 44_278_287_448_695_240, + 47_060_086_756_856_400, + 50_016_653_605_425_536, + 53_158_967_827_883_320, + 56_498_699_069_691_424, + 60_048_250_125_977_912, + 63_820_803_001_928_304, + 67_830_367_866_937_216, + 72_091_835_084_322_176, + 76_621_030_509_822_880, + 81_434_774_264_248_528, + 86_550_943_198_537_824, + 91_988_537_283_208_848, + 97_767_750_168_749_840, + 103_910_044_178_992_000, + 110_438_230_015_967_792, + 117_376_551_472_255_616, + 124_750_775_465_407_920, + 132_588_287_728_824_640, + 140_918_194_514_440_064, + 149_771_430_684_917_568, + 159_180_874_596_775_264, + 169_181_470_201_085_280, + 179_810_356_815_193_344, + 191_107_007_047_393_216, + 203_113_373_386_768_288, + 215_874_044_002_592_672, + 229_436_408_331_885_600, + 243_850_833_070_063_392, + 259_170_849_218_267_264, + 275_453_350_882_006_752, + 292_758_806_559_399_232, + 311_151_483_703_668_992, + 330_699_687_393_865_920, + 351_476_014_000_157_824, + 373_557_620_785_735_808, + 397_026_512_446_556_096, + 421_969_845_653_044_224, + 448_480_252_724_740_928, + 476_656_185_639_923_904, + 506_602_281_657_757_760, + 538_429_751_910_786_752, + 572_256_794_410_890_176, + 608_209_033_002_485_632, + 646_419_983_893_124_352, + 687_031_551_494_039_552, + 730_194_555_412_054_016, + 776_069_290_549_944_960, + 824_826_122_395_314_176, + 876_646_119_708_695_936, + 931_721_726_960_522_368, + 990_257_479_014_182_144, + 1_052_470_760_709_299_712, + 1_118_592_614_166_106_112, + 1_188_868_596_808_997_376, + 1_263_559_693_295_730_432, + 1_342_943_284_738_898_688, + 1_427_314_178_819_094_784, + 1_516_985_704_615_302_400, + 1_612_290_876_218_400_768, + 1_713_583_629_449_105_408, + 1_821_240_136_273_157_632, + 1_935_660_201_795_120_128, + 2_057_268_749_018_809_600, + 2_186_517_396_888_336_384, + 2_323_886_137_470_138_880, + 2_469_885_118_504_583_168, + 2_625_056_537_947_004_416, + 2_789_976_657_533_970_944, + 2_965_257_942_852_572_160, + 3_151_551_337_860_326_400, + 3_349_548_682_302_620_672, + 3_559_985_281_005_267_968, + 3_783_642_634_583_792_128, + 4_021_351_341_710_503_936, + 4_273_994_183_717_548_544, + 4_542_509_402_991_247_872, + 4_827_894_187_332_742_144, + 5_131_208_373_224_844_288, + 5_453_578_381_757_959_168, + 5_796_201_401_831_965_696, + 6_160_349_836_169_256_960, + 6_547_376_026_650_146_816, + 6_958_717_276_519_173_120, + 7_395_901_188_113_309_696, + 7_860_551_335_934_872_576, + 8_354_393_296_137_270_272, + 8_879_261_054_815_360_000, + 9_437_103_818_898_946_048, + 10_029_993_254_943_105_024, + 10_660_131_182_698_121_216, + 11_329_857_752_030_707_712, + 12_041_660_133_563_240_448, + 12_798_181_755_305_525_248, + 13_602_232_119_581_272_064, + 14_456_797_236_706_498_560, + 15_365_050_714_167_523_328, + 16_330_365_542_480_556_032, + 17_356_326_621_502_140_416, + 18_446_744_073_709_551_615, +]; diff --git a/bin/node/runtime/voter-bags/Cargo.toml b/bin/node/runtime/voter-bags/Cargo.toml new file mode 100644 index 0000000000000..38f58ba9d4bd0 --- /dev/null +++ b/bin/node/runtime/voter-bags/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "node-runtime-voter-bags" +version = "3.0.0" +authors = ["Parity Technologies "] +edition = "2018" +license = "Apache-2.0" +homepage = "https://substrate.dev" +repository = "https://github.com/paritytech/substrate/" +description = "Voter Bag generation script for pallet-staking and node-runtime." +readme = "README.md" + +[dependencies] +node-runtime = { version = "2.0.0", path = ".." } +pallet-staking = { version = "3.0.0", path = "../../../../frame/staking", features = ["make-bags"] } +sp-io = { version = "3.0.0", path = "../../../../primitives/io" } +structopt = "0.3.21" diff --git a/bin/node/runtime/voter-bags/src/main.rs b/bin/node/runtime/voter-bags/src/main.rs new file mode 100644 index 0000000000000..fbd86adb185e0 --- /dev/null +++ b/bin/node/runtime/voter-bags/src/main.rs @@ -0,0 +1,68 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Make the set of voting bag thresholds to be used in `voter_bags.rs`. + +use pallet_staking::voter_bags::make_bags::generate_thresholds_module; +use pallet_staking::mock::Test; +use std::path::{Path, PathBuf}; +use structopt::{StructOpt, clap::arg_enum}; + +arg_enum!{ + #[derive(Debug)] + enum Runtime { + Node, + StakingMock, + } +} + +impl Runtime { + fn generate_thresholds(&self) -> Box Result<(), std::io::Error>> { + match self { + Runtime::Node => Box::new(generate_thresholds_module::), + Runtime::StakingMock => Box::new(generate_thresholds_module::), + } + } +} + +#[derive(Debug, StructOpt)] +struct Opt { + /// How many bags to generate. + #[structopt( + long, + default_value = "200", + )] + n_bags: usize, + + /// Which runtime to generate. + #[structopt( + long, + case_insensitive = true, + default_value = "Node", + possible_values = &Runtime::variants(), + )] + runtime: Runtime, + + /// Where to write the output. + output: PathBuf, +} + +fn main() -> Result<(), std::io::Error> { + let Opt {n_bags, output, runtime } = Opt::from_args(); + let mut ext = sp_io::TestExternalities::new_empty(); + ext.execute_with(|| runtime.generate_thresholds()(n_bags, &output)) +} diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 6c1cc89cf1ed0..3a18c11616c05 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -215,6 +215,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_offences::Config for Test { diff --git a/frame/grandpa/src/mock.rs b/frame/grandpa/src/mock.rs index ebe5996c9dab5..27f7b3d488437 100644 --- a/frame/grandpa/src/mock.rs +++ b/frame/grandpa/src/mock.rs @@ -221,6 +221,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_offences::Config for Test { diff --git a/frame/offences/benchmarking/src/mock.rs b/frame/offences/benchmarking/src/mock.rs index cd72780ec5ad2..84527ea4ab033 100644 --- a/frame/offences/benchmarking/src/mock.rs +++ b/frame/offences/benchmarking/src/mock.rs @@ -180,6 +180,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl pallet_im_online::Config for Test { diff --git a/frame/session/benchmarking/src/mock.rs b/frame/session/benchmarking/src/mock.rs index 591e54f067bb5..6415dd8a9fede 100644 --- a/frame/session/benchmarking/src/mock.rs +++ b/frame/session/benchmarking/src/mock.rs @@ -185,6 +185,7 @@ impl pallet_staking::Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = (); } impl crate::Config for Test {} diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index 3e080f2537014..d4d25c86c3940 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -16,14 +16,19 @@ targets = ["x86_64-unknown-linux-gnu"] static_assertions = "1.1.0" serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "2.0.0", default-features = false, features = ["derive"] } +sp-core = { version = "3.0.0", path = "../../primitives/core", optional = true } sp-std = { version = "3.0.0", default-features = false, path = "../../primitives/std" } sp-io ={ version = "3.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "3.0.0", default-features = false, path = "../../primitives/runtime" } sp-staking = { version = "3.0.0", default-features = false, path = "../../primitives/staking" } +sp-tracing = { version = "3.0.0", path = "../../primitives/tracing", optional = true } frame-support = { version = "3.0.0", default-features = false, path = "../support" } frame-system = { version = "3.0.0", default-features = false, path = "../system" } -pallet-session = { version = "3.0.0", default-features = false, features = ["historical"], path = "../session" } pallet-authorship = { version = "3.0.0", default-features = false, path = "../authorship" } +pallet-balances = { version = "3.0.0", path = "../balances", optional = true } +pallet-session = { version = "3.0.0", default-features = false, features = ["historical"], path = "../session" } +pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve", optional = true } +pallet-timestamp = { version = "3.0.0", path = "../timestamp", optional = true } sp-application-crypto = { version = "3.0.0", default-features = false, path = "../../primitives/application-crypto" } frame-election-provider-support = { version = "3.0.0", default-features = false, path = "../election-provider-support" } log = { version = "0.4.14", default-features = false } @@ -33,6 +38,11 @@ paste = "1.0" frame-benchmarking = { version = "3.1.0", default-features = false, path = "../benchmarking", optional = true } rand_chacha = { version = "0.2", default-features = false, optional = true } +# Optional imports for making voter bags lists +chrono = { version = "0.4.19", optional = true } +git2 = { version = "0.13.20", default-features = false, optional = true } +num-format = { version = "0.4.0", optional = true } + [dev-dependencies] sp-storage = { version = "3.0.0", path = "../../primitives/storage" } sp-tracing = { version = "3.0.0", path = "../../primitives/tracing" } @@ -40,7 +50,7 @@ sp-core = { version = "3.0.0", path = "../../primitives/core" } sp-npos-elections = { version = "3.0.0", path = "../../primitives/npos-elections", features = ["mocks"] } pallet-balances = { version = "3.0.0", path = "../balances" } pallet-timestamp = { version = "3.0.0", path = "../timestamp" } -pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } +pallet-staking-reward-curve = { version = "3.0.0", path = "../staking/reward-curve" } substrate-test-utils = { version = "3.0.0", path = "../../test-utils" } frame-benchmarking = { version = "3.1.0", path = "../benchmarking" } frame-election-provider-support = { version = "3.0.0", features = ["runtime-benchmarks"], path = "../election-provider-support" } @@ -71,8 +81,14 @@ runtime-benchmarks = [ "rand_chacha", ] try-runtime = ["frame-support/try-runtime"] -make-bags = [] - -[[bin]] -name = "make_bags" -required-features = ["make-bags", "std"] +make-bags = [ + "chrono", + "git2", + "num-format", + "pallet-staking-reward-curve", + "pallet-balances", + "pallet-timestamp", + "sp-core", + "sp-tracing", + "std", +] diff --git a/frame/staking/src/benchmarking.rs b/frame/staking/src/benchmarking.rs index 208769ce1ccac..fbae528a95058 100644 --- a/frame/staking/src/benchmarking.rs +++ b/frame/staking/src/benchmarking.rs @@ -700,7 +700,7 @@ benchmarks! { ); ensure!( { - let origin_bag = Bag::::get(node.bag_idx).ok_or("origin bag not found")?; + let origin_bag = Bag::::get(node.bag_upper).ok_or("origin bag not found")?; origin_bag.iter().count() == 1 }, "stash should be the only node in origin bag", diff --git a/frame/staking/src/bin/make_bags.rs b/frame/staking/src/bin/make_bags.rs deleted file mode 100644 index 664c7b5470ba0..0000000000000 --- a/frame/staking/src/bin/make_bags.rs +++ /dev/null @@ -1,40 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Make the set of voting bag thresholds to be used in `voter_bags.rs`. - -use pallet_staking::voter_bags::N_BAGS; - -fn main() { - let ratio = ((u64::MAX as f64).ln() / (N_BAGS as f64)).exp(); - println!("pub const CONSTANT_RATIO: f64 = {};", ratio); - - let mut thresholds = Vec::with_capacity(N_BAGS as usize); - - while thresholds.len() < N_BAGS as usize { - let prev_item: u64 = thresholds.last().copied().unwrap_or_default(); - let item = (prev_item as f64 * ratio).max(prev_item as f64 + 1.0); - thresholds.push(item as u64); - } - - *thresholds.last_mut().unwrap() = u64::MAX; - - println!("pub const THRESHOLDS: [u64; {}] = {:#?};", N_BAGS, thresholds); - - debug_assert_eq!(thresholds.len(), N_BAGS as usize); - debug_assert_eq!(*thresholds.last().unwrap(), u64::MAX); -} diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 0abd609db6dfd..0c761e2528bd6 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -274,8 +274,8 @@ #![recursion_limit = "128"] #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(test)] -mod mock; +#[cfg(any(test, feature = "make-bags"))] +pub mod mock; #[cfg(test)] mod tests; #[cfg(any(feature = "runtime-benchmarks", test))] @@ -308,11 +308,11 @@ use frame_support::{ }; use pallet_session::historical; use sp_runtime::{ - Percent, Perbill, RuntimeDebug, DispatchError, + DispatchError, Perbill, Percent, RuntimeDebug, curve::PiecewiseLinear, traits::{ - Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, - AtLeast32BitUnsigned, Bounded, + AtLeast32BitUnsigned, Bounded, CheckedSub, Convert, SaturatedConversion, Saturating, + StaticLookup, Zero, }, }; use sp_staking::{ @@ -321,7 +321,7 @@ use sp_staking::{ }; use frame_system::{ensure_signed, ensure_root, pallet_prelude::*, offchain::SendTransactionTypes}; use frame_election_provider_support::{ElectionProvider, VoteWeight, Supports, data_provider}; -use voter_bags::{BagIdx, VoterList, VoterType}; +use voter_bags::{VoterList, VoterType}; pub use weights::WeightInfo; pub use pallet::*; @@ -960,6 +960,56 @@ pub mod pallet { /// Weight information for extrinsics in this pallet. type WeightInfo: WeightInfo; + + /// The list of thresholds separating the various voter bags. + /// + /// Voters are separated into unsorted bags according to their vote weight. This specifies + /// the thresholds separating the bags. A voter's bag is the largest bag for which the + /// voter's weight is less than or equal to its upper threshold. + /// + /// When voters are iterated, higher bags are iterated completely before lower bags. This + /// means that iteration is _semi-sorted_: voters of higher weight tend to come before + /// voters of lower weight, but peer voters within a particular bag are sorted in insertion + /// order. + /// + /// # Expressing the constant + /// + /// This constant must be sorted in strictly increasing order. Duplicate items are not + /// permitted. + /// + /// There is an implied upper limit of `VoteWeight::MAX`; that value does not need to be + /// specified within the bag. For any two threshold lists, if one ends with + /// `VoteWeight::MAX`, the other one does not, and they are otherwise equal, the two lists + /// will behave identically. + /// + /// # Calculation + /// + /// It is recommended to generate the set of thresholds in a geometric series, such that + /// there exists some constant ratio such that `threshold[k + 1] == (threshold[k] * + /// constant_ratio).max(threshold[k] + 1)` for all `k`. + /// + /// The helpers in the `voter_bags::make_bags` module can simplify this calculation. To use + /// them, the `make-bags` feature must be enabled. + /// + /// # Examples + /// + /// - If `VoterBagThresholds::get().is_empty()`, then all voters are put into the same bag, + /// and iteration is strictly in insertion order. + /// - If `VoterBagThresholds::get().len() == 64`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is equal to 2. + /// - If `VoterBagThresholds::get().len() == 200`, and the thresholds are determined + /// according to the procedure given above, then the constant ratio is approximately equal + /// to 1.248. + /// - If the threshold list begins `[1, 2, 3, ...]`, then a voter with weight 0 or 1 will + /// fall into bag 0, a voter with weight 2 will fall into bag 1, etc. + /// + /// # Migration + /// + /// In the event that this list ever changes, a copy of the old bags list must be retained. + /// With that `VoterList::migrate` can be called, which will perform the appropriate + /// migration. + #[pallet::constant] + type VoterBagThresholds: Get<&'static [VoteWeight]>; } #[pallet::extra_constants] @@ -1258,6 +1308,8 @@ pub mod pallet { // The next storage items collectively comprise the voter bags: a composite data structure // designed to allow efficient iteration of the top N voters by stake, mostly. See // `mod voter_bags` for details. + // + // In each of these items, voter bags are indexed by their upper weight threshold. /// How many voters are registered. #[pallet::storage] @@ -1268,21 +1320,25 @@ pub mod pallet { /// This may not be the appropriate bag for the voter's weight if they have been rewarded or /// slashed. #[pallet::storage] - pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, voter_bags::BagIdx>; + pub(crate) type VoterBagFor = StorageMap<_, Twox64Concat, AccountIdOf, VoteWeight>; - /// The head and tail of each bag of voters. + /// This storage item maps a bag (identified by its upper threshold) to the `Bag` struct, which + /// mainly exists to store head and tail pointers to the appropriate nodes. #[pallet::storage] pub(crate) type VoterBags = StorageMap< _, - Twox64Concat, voter_bags::BagIdx, + Twox64Concat, VoteWeight, voter_bags::Bag, >; - /// The nodes comprising each bag. + /// Voter nodes store links forward and back within their respective bags, the stash id, and + /// whether the voter is a validator or nominator. + /// + /// There is nothing in this map directly identifying to which bag a particular node belongs. + /// However, the `Node` data structure has helpers which can provide that information. #[pallet::storage] - pub(crate) type VoterNodes = StorageDoubleMap< + pub(crate) type VoterNodes = StorageMap< _, - Twox64Concat, voter_bags::BagIdx, Twox64Concat, AccountIdOf, voter_bags::Node, >; @@ -1403,7 +1459,7 @@ pub mod pallet { /// The election failed. No new era is planned. StakingElectionFailed, /// Moved an account from one bag to another. \[who, from, to\]. - Rebagged(T::AccountId, BagIdx, BagIdx), + Rebagged(T::AccountId, VoteWeight, VoteWeight), } #[pallet::error] @@ -1497,14 +1553,19 @@ pub mod pallet { fn integrity_test() { sp_std::if_std! { - sp_io::TestExternalities::new_empty().execute_with(|| + sp_io::TestExternalities::new_empty().execute_with(|| { assert!( T::SlashDeferDuration::get() < T::BondingDuration::get() || T::BondingDuration::get() == 0, "As per documentation, slash defer duration ({}) should be less than bonding duration ({}).", T::SlashDeferDuration::get(), T::BondingDuration::get(), - ) - ); + ); + + assert!( + T::VoterBagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "Voter bag thresholds must strictly increase", + ); + }); } } } @@ -3141,7 +3202,7 @@ impl Pallet { /// Move a stash account from one bag to another, depositing an event on success. /// /// If the stash changed bags, returns `Some((from, to))`. - pub fn do_rebag(stash: &T::AccountId) -> Option<(BagIdx, BagIdx)> { + pub fn do_rebag(stash: &T::AccountId) -> Option<(VoteWeight, VoteWeight)> { // if no voter at that node, don't do anything. // the caller just wasted the fee to call this. let maybe_movement = voter_bags::Node::::from_id(&stash).and_then(|node| { diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index e0079cc3f375a..a584b83c539c8 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -17,6 +17,12 @@ //! Test utilities +// This module needs to exist when the `make-bags` feature is enabled so that we can generate the +// appropriate thresholds, but we don't care if it's mostly unused in that case. +#![cfg_attr(feature = "make-bags", allow(unused))] + +mod voter_bags; + use crate::*; use crate as staking; use frame_support::{ @@ -243,6 +249,10 @@ impl onchain::Config for Test { type DataProvider = Staking; } +parameter_types! { + pub const VoterBagThresholds: &'static [VoteWeight] = &voter_bags::THRESHOLDS; +} + impl Config for Test { const MAX_NOMINATIONS: u32 = 16; type Currency = Balances; @@ -263,6 +273,7 @@ impl Config for Test { type ElectionProvider = onchain::OnChainSequentialPhragmen; type GenesisElectionProvider = Self::ElectionProvider; type WeightInfo = (); + type VoterBagThresholds = VoterBagThresholds; } impl frame_system::offchain::SendTransactionTypes for Test diff --git a/frame/staking/src/mock/voter_bags.rs b/frame/staking/src/mock/voter_bags.rs new file mode 100644 index 0000000000000..453b03e36ab22 --- /dev/null +++ b/frame/staking/src/mock/voter_bags.rs @@ -0,0 +1,100 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Autogenerated voter bag thresholds. +//! +//! Generated on 2021-07-05T12:08:52.871368217+00:00 +//! for the test runtime. + +/// Existential weight for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const EXISTENTIAL_WEIGHT: u64 = 1; + +/// Constant ratio between bags for this runtime. +#[cfg(any(test, feature = "std"))] +#[allow(unused)] +pub const CONSTANT_RATIO: f64 = 2.0000000000000000; + +/// Upper thresholds delimiting the bag list. +pub const THRESHOLDS: [u64; 65] = [ + 1, + 2, + 4, + 8, + 16, + 32, + 64, + 128, + 256, + 512, + 1_024, + 2_048, + 4_096, + 8_192, + 16_384, + 32_768, + 65_536, + 131_072, + 262_144, + 524_288, + 1_048_576, + 2_097_152, + 4_194_304, + 8_388_608, + 16_777_216, + 33_554_432, + 67_108_864, + 134_217_728, + 268_435_456, + 536_870_912, + 1_073_741_824, + 2_147_483_648, + 4_294_967_296, + 8_589_934_592, + 17_179_869_184, + 34_359_738_368, + 68_719_476_736, + 137_438_953_472, + 274_877_906_944, + 549_755_813_888, + 1_099_511_627_776, + 2_199_023_255_552, + 4_398_046_511_104, + 8_796_093_022_208, + 17_592_186_044_416, + 35_184_372_088_832, + 70_368_744_177_664, + 140_737_488_355_328, + 281_474_976_710_656, + 562_949_953_421_312, + 1_125_899_906_842_624, + 2_251_799_813_685_248, + 4_503_599_627_370_496, + 9_007_199_254_740_992, + 18_014_398_509_481_984, + 36_028_797_018_963_968, + 72_057_594_037_927_936, + 144_115_188_075_855_872, + 288_230_376_151_711_744, + 576_460_752_303_423_488, + 1_152_921_504_606_846_976, + 2_305_843_009_213_693_952, + 4_611_686_018_427_387_904, + 9_223_372_036_854_775_808, + 18_446_744_073_709_551_615, +]; diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index e0e6b83d4b742..864fcb0c35683 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -3909,7 +3909,7 @@ fn test_rebag() { let node = Node::::from_id(&stash).unwrap(); assert_eq!( { - let origin_bag = Bag::::get(node.bag_idx).unwrap(); + let origin_bag = Bag::::get(node.bag_upper).unwrap(); origin_bag.iter().count() }, 1, @@ -3919,7 +3919,7 @@ fn test_rebag() { assert!(!other_node.is_misplaced(&weight_of), "other stash balance never changed"); assert_ne!( { - let destination_bag = Bag::::get(other_node.bag_idx); + let destination_bag = Bag::::get(other_node.bag_upper); destination_bag.iter().count() }, 0, diff --git a/frame/staking/src/voter_bags.rs b/frame/staking/src/voter_bags.rs new file mode 100644 index 0000000000000..1d748142f6ce7 --- /dev/null +++ b/frame/staking/src/voter_bags.rs @@ -0,0 +1,807 @@ +// This file is part of Substrate. + +// Copyright (C) 2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implement a data structure designed for the properties that: +//! +//! - It's efficient to insert or remove a voter +//! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of +//! voters doesn't particularly matter. + +use crate::{ + AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, + VotingDataOf, slashing::SlashingSpans, +}; +use codec::{Encode, Decode}; +use frame_support::{DefaultNoBound, traits::Get}; +use sp_runtime::SaturatedConversion; +use sp_std::{collections::{btree_map::BTreeMap, btree_set::BTreeSet}, marker::PhantomData}; + +/// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. +pub type VoterOf = Voter>; + +/// Given a certain vote weight, which bag should contain this voter? +/// +/// Bags are identified by their upper threshold; the value returned by this function is guaranteed +/// to be a member of `T::VoterBagThresholds`. +/// +/// This is used instead of a simpler scheme, such as the index within `T::VoterBagThresholds`, +/// because in the event that bags are inserted or deleted, the number of affected voters which need +/// to be migrated is smaller. +/// +/// Note that even if the thresholds list does not have `VoteWeight::MAX` as its final member, this +/// function behaves as if it does. +fn notional_bag_for(weight: VoteWeight) -> VoteWeight { + let thresholds = T::VoterBagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| weight > threshold); + thresholds.get(idx).copied().unwrap_or(VoteWeight::MAX) +} + +/// Find the upper threshold of the actual bag containing the current voter. +fn current_bag_for(id: &AccountIdOf) -> Option { + VoterBagFor::::try_get(id).ok() +} + +/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. +/// +/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of +/// arbitrary and unbounded length, all having a vote weight within a particular constant range. +/// This structure means that voters can be added and removed in `O(1)` time. +/// +/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. +/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall +/// stake decreases as successive bags are reached. This means that it is valid to truncate +/// iteration at any desired point; only those voters in the lowest bag (who are known to have +/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire +/// for fairness and the requirement for efficiency. +pub struct VoterList(PhantomData); + +impl VoterList { + /// Remove all data associated with the voter list from storage. + pub fn clear() { + crate::VoterCount::::kill(); + crate::VoterBagFor::::remove_all(None); + crate::VoterBags::::remove_all(None); + crate::VoterNodes::::remove_all(None); + } + + /// Regenerate voter data from the `Nominators` and `Validators` storage items. + /// + /// This is expensive and should only ever be performed during a migration, never during + /// consensus. + /// + /// Returns the number of voters migrated. + pub fn regenerate() -> u32 { + Self::clear(); + + let nominators_iter = Nominators::::iter().map(|(id, _)| Voter::nominator(id)); + let validators_iter = Validators::::iter().map(|(id, _)| Voter::validator(id)); + let weight_of = Pallet::::weight_of_fn(); + + Self::insert_many( + nominators_iter.chain(validators_iter), + weight_of, + ) + } + + /// Decode the length of the voter list. + pub fn decode_len() -> Option { + crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) + } + + /// Iterate over all nodes in all bags in the voter list. + /// + /// Full iteration can be expensive; it's recommended to limit the number of items with `.take(n)`. + pub fn iter() -> impl Iterator> { + T::VoterBagThresholds::get() + .iter() + .rev() + .copied() + .filter_map(Bag::get) + .flat_map(|bag| bag.iter()) + } + + /// Insert a new voter into the appropriate bag in the voter list. + /// + /// If the voter is already present in the list, their type will be updated. + /// That case is cheaper than inserting a new voter. + pub fn insert_as(account_id: &AccountIdOf, voter_type: VoterType) { + // if this is an update operation we can complete this easily and cheaply + if !Node::::update_voter_type_for(account_id, voter_type) { + // otherwise, we need to insert from scratch + let weight_of = Pallet::::weight_of_fn(); + let voter = Voter { id: account_id.clone(), voter_type }; + Self::insert(voter, weight_of); + } + } + + /// Insert a new voter into the appropriate bag in the voter list. + pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { + Self::insert_many(sp_std::iter::once(voter), weight_of); + } + + /// Insert several voters into the appropriate bags in the voter list. + /// + /// This is more efficient than repeated calls to `Self::insert`. + pub fn insert_many( + voters: impl IntoIterator>, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + ) -> u32 { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for voter in voters.into_iter() { + let weight = weight_of(&voter.id); + let bag = notional_bag_for::(weight); + bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); + count += 1; + } + + for (_, bag) in bags { + bag.put(); + } + + crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); + count + } + + /// Remove a voter (by id) from the voter list. + pub fn remove(voter: &AccountIdOf) { + Self::remove_many(sp_std::iter::once(voter)) + } + + /// Remove many voters (by id) from the voter list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + pub fn remove_many<'a>(voters: impl IntoIterator>) { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for voter_id in voters.into_iter() { + let node = match Node::::from_id(voter_id) { + Some(node) => node, + None => continue, + }; + count += 1; + + // clear the bag head/tail pointers as necessary + let bag = bags.entry(node.bag_upper).or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + bag.remove_node(&node); + + // now get rid of the node itself + crate::VoterNodes::::remove(voter_id); + crate::VoterBagFor::::remove(voter_id); + } + + for (_, bag) in bags { + bag.put(); + } + + crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_sub(count)); + } + + /// Update a voter's position in the voter list. + /// + /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `true` if the voter moved. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub fn update_position_for( + mut node: Node, + weight_of: impl Fn(&AccountIdOf) -> VoteWeight, + ) -> Option<(VoteWeight, VoteWeight)> { + node.is_misplaced(&weight_of).then(move || { + let old_idx = node.bag_upper; + + // clear the old bag head/tail pointers as necessary + if let Some(mut bag) = Bag::::get(node.bag_upper) { + bag.remove_node(&node); + bag.put(); + } else { + debug_assert!(false, "every node must have an extant bag associated with it"); + crate::log!( + error, + "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", + node.voter.id, + ); + } + + // put the voter into the appropriate new bag + let new_idx = notional_bag_for::(weight_of(&node.voter.id)); + node.bag_upper = new_idx; + let mut bag = Bag::::get_or_make(node.bag_upper); + bag.insert_node(node); + bag.put(); + + (old_idx, new_idx) + }) + } + + /// Migrate the voter list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::VoterBagThresholds` has already been updated. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::VoterBagThresholds`. + /// - No voter is changed unless required to by the difference between the old threshold list + /// and the new. + /// - Voters whose bags change at all are implicitly rebagged into the appropriate bag in the + /// new threshold set. + pub fn migrate(old_thresholds: &[VoteWeight]) -> u32 { + // we can't check all preconditions, but we can check one + debug_assert!( + crate::VoterBags::::iter().all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = T::VoterBagThresholds::get().iter().copied().collect(); + + let mut affected_accounts = BTreeSet::new(); + let mut affected_old_bags = BTreeSet::new(); + + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_set.difference(&old_set).copied() { + let affected_bag = notional_bag_for::(inserted_bag); + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // a removed bag means that all members of that bag must be rebagged + for removed_bag in old_set.difference(&new_set).copied() { + if !affected_old_bags.insert(removed_bag) { + continue + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.voter)); + } + } + + // migrate the + let weight_of = Pallet::::weight_of_fn(); + Self::remove_many(affected_accounts.iter().map(|voter| &voter.id)); + let num_affected = Self::insert_many(affected_accounts.into_iter(), weight_of); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in old_set.difference(&new_set).copied() { + debug_assert!( + !VoterBagFor::::iter().any(|(_voter, bag)| bag == removed_bag), + "no voter should be present in a removed bag", + ); + crate::VoterBags::::remove(removed_bag); + } + + debug_assert!( + { + let thresholds = T::VoterBagThresholds::get(); + crate::VoterBags::::iter().all(|(threshold, _)| thresholds.contains(&threshold)) + }, + "all `bag_upper` in storage must be members of the new thresholds", + ); + + num_affected + } +} + +/// A Bag is a doubly-linked list of voters. +/// +/// Note that we maintain both head and tail pointers. While it would be possible to get away +/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's +/// more desirable to ensure that there is some element of first-come, first-serve to the list's +/// iteration so that there's no incentive to churn voter positioning to improve the chances of +/// appearing within the voter set. +#[derive(DefaultNoBound, Encode, Decode)] +pub struct Bag { + head: Option>, + tail: Option>, + + #[codec(skip)] + bag_upper: VoteWeight, +} + +impl Bag { + /// Get a bag by its upper vote weight. + pub fn get(bag_upper: VoteWeight) -> Option> { + debug_assert!( + T::VoterBagThresholds::get().contains(&bag_upper) || bag_upper == VoteWeight::MAX, + "it is a logic error to attempt to get a bag which is not in the thresholds list" + ); + crate::VoterBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; + bag + }) + } + + /// Get a bag by its upper vote weight or make it, appropriately initialized. + pub fn get_or_make(bag_upper: VoteWeight) -> Bag { + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) + } + + /// Put the bag back into storage. + pub fn put(self) { + crate::VoterBags::::insert(self.bag_upper, self); + } + + /// Get the head node in this bag. + pub fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + } + + /// Get the tail node in this bag. + pub fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(self.bag_upper, id)) + } + + /// Iterate over the nodes in this bag. + pub fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Insert a new voter into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + fn insert(&mut self, voter: VoterOf) { + self.insert_node(Node:: { + voter, + prev: None, + next: None, + bag_upper: self.bag_upper, + }); + } + + /// Insert a voter node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + fn insert_node(&mut self, mut node: Node) { + let id = node.voter.id.clone(); + + node.prev = self.tail.clone(); + node.next = None; + node.put(); + + // update the previous tail + if let Some(mut tail) = self.tail() { + tail.next = Some(id.clone()); + tail.put(); + } + + // update the internal bag links + if self.head.is_none() { + self.head = Some(id.clone()); + } + self.tail = Some(id.clone()); + + crate::VoterBagFor::::insert(id, self.bag_upper); + } + + /// Remove a voter node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the voter in + /// the first place. Generally, use [`VoterList::remove`] instead. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()` and `node.put()` after use. + fn remove_node(&mut self, node: &Node) { + node.excise(); + + // clear the bag head/tail pointers as necessary + if self.head.as_ref() == Some(&node.voter.id) { + self.head = node.next.clone(); + } + if self.tail.as_ref() == Some(&node.voter.id) { + self.tail = node.prev.clone(); + } + } +} + +/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. +#[derive(Encode, Decode)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Node { + voter: Voter>, + prev: Option>, + next: Option>, + + /// The bag index is not stored in storage, but injected during all fetch operations. + #[codec(skip)] + pub(crate) bag_upper: VoteWeight, +} + +impl Node { + /// Get a node by bag idx and account id. + pub fn get(bag_upper: VoteWeight, account_id: &AccountIdOf) -> Option> { + crate::VoterNodes::::try_get(account_id).ok().map(|mut node| { + node.bag_upper = bag_upper; + node + }) + } + + /// Get a node by account id. + /// + /// Note that this must perform two storage lookups: one to identify which bag is appropriate, + /// and another to actually fetch the node. + pub fn from_id(account_id: &AccountIdOf) -> Option> { + let bag = current_bag_for::(account_id)?; + Self::get(bag, account_id) + } + + /// Get a node by account id, assuming it's in the same bag as this node. + pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { + Self::get(self.bag_upper, account_id) + } + + /// Put the node back into storage. + pub fn put(self) { + crate::VoterNodes::::insert(self.voter.id.clone(), self); + } + + /// Get the previous node in the bag. + pub fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| self.in_bag(id)) + } + + /// Get the next node in the bag. + pub fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| self.in_bag(id)) + } + + /// Get this voter's voting data. + pub fn voting_data( + &self, + weight_of: impl Fn(&T::AccountId) -> VoteWeight, + slashing_spans: &BTreeMap, SlashingSpans>, + ) -> Option> { + match self.voter.voter_type { + VoterType::Validator => Some(( + self.voter.id.clone(), + weight_of(&self.voter.id), + sp_std::vec![self.voter.id.clone()], + )), + VoterType::Nominator => { + let Nominations { submitted_in, mut targets, .. } = + Nominators::::get(self.voter.id.clone())?; + // Filter out nomination targets which were nominated before the most recent + // slashing span. + targets.retain(|stash| { + slashing_spans + .get(stash) + .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) + }); + + (!targets.is_empty()) + .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) + } + } + } + + /// Remove this node from the linked list. + /// + /// Modifies storage, but only modifies the adjacent nodes. Does not modify `self` or any bag. + fn excise(&self) { + if let Some(mut prev) = self.prev() { + prev.next = self.next.clone(); + prev.put(); + } + if let Some(mut next) = self.next() { + next.prev = self.prev.clone(); + next.put(); + } + } + + /// `true` when this voter is in the wrong bag. + pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { + notional_bag_for::(weight_of(&self.voter.id)) != self.bag_upper + } + + /// Update the voter type associated with a particular node by id. + /// + /// This updates storage immediately. + /// + /// Returns whether the voter existed and was successfully updated. + pub fn update_voter_type_for(account_id: &AccountIdOf, voter_type: VoterType) -> bool { + let node = Self::from_id(account_id); + let existed = node.is_some(); + if let Some(mut node) = node { + node.voter.voter_type = voter_type; + node.put(); + } + existed + } + + /// Get the upper threshold of the bag that this node _should_ be in, given its vote weight. + /// + /// This is a helper intended only for benchmarking and should not be used in production. + #[cfg(any(test, feature = "runtime-benchmarks"))] + pub fn proper_bag_for(&self) -> VoteWeight { + let weight_of = crate::Pallet::::weight_of_fn(); + let current_weight = weight_of(&self.voter.id); + notional_bag_for::(current_weight) + } +} + +/// Fundamental information about a voter. +#[derive(Clone, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr(feature = "std", derive(Debug))] +pub struct Voter { + /// Account Id of this voter + pub id: AccountId, + /// Whether the voter is a validator or nominator + pub voter_type: VoterType, +} + +impl Voter { + pub fn nominator(id: AccountId) -> Self { + Self { + id, + voter_type: VoterType::Nominator, + } + } + + pub fn validator(id: AccountId) -> Self { + Self { + id, + voter_type: VoterType::Validator, + } + } +} + +/// Type of voter. +/// +/// Similar to [`crate::StakerStatus`], but somewhat more limited. +#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq, PartialOrd, Ord)] +#[cfg_attr(feature = "std", derive(Debug))] +pub enum VoterType { + Validator, + Nominator, +} + +/// Support code to ease the process of generating voter bags. +/// +/// The process of adding voter bags to a runtime requires only four steps. +/// +/// 1. Update the runtime definition. +/// +/// ```ignore +/// parameter_types!{ +/// pub const VoterBagThresholds: &'static [u64] = &[]; +/// } +/// +/// impl pallet_staking::Config for Runtime { +/// // +/// type VoterBagThresholds = VoterBagThresholds; +/// } +/// ``` +/// +/// 2. Write a little program to generate the definitions. This can be a near-identical copy of +/// `substrate/node/runtime/voter-bags`. This program exists only to hook together the runtime +/// definitions with the various calculations here. +/// +/// 3. Run that program: +/// +/// ```sh,notrust +/// $ cargo run -p node-runtime-voter-bags -- bin/node/runtime/src/voter_bags.rs +/// ``` +/// +/// 4. Update the runtime definition. +/// +/// ```diff,notrust +/// + mod voter_bags; +/// - pub const VoterBagThresholds: &'static [u64] = &[]; +/// + pub const VoterBagThresholds: &'static [u64] = &voter_bags::THRESHOLDS; +/// ``` +#[cfg(feature = "make-bags")] +pub mod make_bags { + use crate::{AccountIdOf, Config}; + use frame_election_provider_support::VoteWeight; + use frame_support::traits::{Currency, CurrencyToVote, Get}; + use std::{io::Write, path::{Path, PathBuf}}; + + /// Return the path to a header file used in this repository if is exists. + /// + /// Just searches the git working directory root for files matching certain patterns; it's + /// pretty naive. + fn path_to_header_file() -> Option { + let repo = git2::Repository::open_from_env().ok()?; + let workdir = repo.workdir()?; + for file_name in ["HEADER-APACHE2", "HEADER-GPL3", "HEADER"] { + let path = workdir.join(file_name); + if path.exists() { + return Some(path); + } + } + None + } + + /// Create an underscore formatter: a formatter which inserts `_` every 3 digits of a number. + fn underscore_formatter() -> num_format::CustomFormat { + num_format::CustomFormat::builder() + .grouping(num_format::Grouping::Standard) + .separator("_") + .build() + .expect("format described here meets all constraints") + } + + /// Compute the existential weight for the specified configuration. + /// + /// Note that this value depends on the current issuance, a quantity known to change over time. + /// This makes the project of computing a static value suitable for inclusion in a static, + /// generated file _excitingly unstable_. + pub fn existential_weight() -> VoteWeight { + let existential_deposit = >>::minimum_balance(); + let issuance = >>::total_issuance(); + T::CurrencyToVote::to_vote(existential_deposit, issuance) + } + + /// Compute the constant ratio for the thresholds. + /// + /// This ratio ensures that each bag, with the possible exceptions of certain small ones and the + /// final one, is a constant multiple of the previous, while fully occupying the `VoteWeight` + /// space. + pub fn constant_ratio(existential_weight: VoteWeight, n_bags: usize) -> f64 { + ((VoteWeight::MAX as f64 / existential_weight as f64).ln() / ((n_bags - 1) as f64)).exp() + } + + /// Compute the list of bag thresholds. + /// + /// Returns a list of exactly `n_bags` elements, except in the case of overflow. + /// The first element is always `existential_weight`. + /// The last element is always `VoteWeight::MAX`. + /// + /// All other elements are computed from the previous according to the formula + /// `threshold[k + 1] = (threshold[k] * ratio).max(threshold[k] + 1); + pub fn thresholds( + existential_weight: VoteWeight, + constant_ratio: f64, + n_bags: usize, + ) -> Vec { + const WEIGHT_LIMIT: f64 = VoteWeight::MAX as f64; + + let mut thresholds = Vec::with_capacity(n_bags); + + if n_bags > 1 { + thresholds.push(existential_weight); + } + + while n_bags > 0 && thresholds.len() < n_bags - 1 { + let last = thresholds.last().copied().unwrap_or(existential_weight); + let successor = (last as f64 * constant_ratio).round().max(last as f64 + 1.0); + if successor < WEIGHT_LIMIT { + thresholds.push(successor as VoteWeight); + } else { + eprintln!("unexpectedly exceeded weight limit; breaking threshold generation loop"); + break + } + } + + thresholds.push(VoteWeight::MAX); + + debug_assert_eq!(thresholds.len(), n_bags); + debug_assert!(n_bags == 0 || thresholds[0] == existential_weight); + debug_assert!(n_bags == 0 || thresholds[thresholds.len() - 1] == VoteWeight::MAX); + + thresholds + } + + /// Write a thresholds module to the path specified. + /// + /// The `output` path should terminate with a Rust module name, i.e. `foo/bar/thresholds.rs`. + /// + /// This generated module contains, in order: + /// + /// - The contents of the header file in this repository's root, if found. + /// - Module documentation noting that this is autogenerated and when. + /// - Some associated constants. + /// - The constant array of thresholds. + pub fn generate_thresholds_module(n_bags: usize, output: &Path) -> Result<(), std::io::Error> { + // ensure the file is accessable + if let Some(parent) = output.parent() { + if !parent.exists() { + std::fs::create_dir_all(parent)?; + } + } + + // copy the header file + if let Some(header_path) = path_to_header_file() { + std::fs::copy(header_path, output)?; + } + + // open an append buffer + let file = std::fs::OpenOptions::new().create(true).append(true).open(output)?; + let mut buf = std::io::BufWriter::new(file); + + // create underscore formatter and format buffer + let mut num_buf = num_format::Buffer::new(); + let format = underscore_formatter(); + + // module docs + let now = chrono::Utc::now(); + writeln!(buf)?; + writeln!(buf, "//! Autogenerated voter bag thresholds.")?; + writeln!(buf, "//!")?; + writeln!(buf, "//! Generated on {}", now.to_rfc3339())?; + writeln!( + buf, + "//! for the {} runtime.", + ::Version::get().spec_name, + )?; + + // existential weight + let existential_weight = existential_weight::(); + num_buf.write_formatted(&existential_weight, &format); + writeln!(buf)?; + writeln!(buf, "/// Existential weight for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const EXISTENTIAL_WEIGHT: u64 = {};", num_buf.as_str())?; + + // constant ratio + let constant_ratio = constant_ratio(existential_weight, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Constant ratio between bags for this runtime.")?; + writeln!(buf, "#[cfg(any(test, feature = \"std\"))]")?; + writeln!(buf, "#[allow(unused)]")?; + writeln!(buf, "pub const CONSTANT_RATIO: f64 = {:.16};", constant_ratio)?; + + // thresholds + let thresholds = thresholds(existential_weight, constant_ratio, n_bags); + writeln!(buf)?; + writeln!(buf, "/// Upper thresholds delimiting the bag list.")?; + writeln!(buf, "pub const THRESHOLDS: [u64; {}] = [", thresholds.len())?; + for threshold in thresholds { + num_buf.write_formatted(&threshold, &format); + // u64::MAX, with spacers every 3 digits, is 26 characters wide + writeln!(buf, " {:>26},", num_buf.as_str())?; + } + writeln!(buf, "];")?; + + Ok(()) + } +} diff --git a/frame/staking/src/voter_bags/mod.rs b/frame/staking/src/voter_bags/mod.rs deleted file mode 100644 index f768f056115b2..0000000000000 --- a/frame/staking/src/voter_bags/mod.rs +++ /dev/null @@ -1,514 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Implement a data structure designed for the properties that: -//! -//! - It's efficient to insert or remove a voter -//! - It's efficient to iterate over the top* N voters by stake, where the precise ordering of -//! voters doesn't particularly matter. - -mod thresholds; - -use thresholds::THRESHOLDS; -use crate::{ - AccountIdOf, Config, Nominations, Nominators, Pallet, Validators, VoteWeight, VoterBagFor, - VotingDataOf, slashing::SlashingSpans, -}; -use codec::{Encode, Decode}; -use frame_support::DefaultNoBound; -use sp_runtime::SaturatedConversion; -use sp_std::{collections::btree_map::BTreeMap, marker::PhantomData}; - -/// [`Voter`] parametrized by [`Config`] instead of by `AccountId`. -pub type VoterOf = Voter>; - -/// Index type for a bag. -pub type BagIdx = u8; - -/// How many bags there are -pub const N_BAGS: BagIdx = 200; - -/// Given a certain vote weight, which bag should this voter contain? -/// -/// Bags are separated by fixed thresholds. To the extent possible, each threshold is a constant -/// small multiple of the one before it. That ratio is [`thresholds::CONSTANT_RATIO`]. The exception -/// are the smallest bags, which are each at least 1 greater than the previous, and the largest bag, -/// which is defined as `u64::MAX`. -/// -/// Bags are arranged such that `bags[0]` is the largest bag, and `bags[N_BAGS-1]` is the smallest. -fn notional_bag_for(weight: VoteWeight) -> BagIdx { - let raw_bag = - THRESHOLDS.partition_point(|&threshold| weight > threshold) as BagIdx; - N_BAGS - 1 - raw_bag -} - -/// Find the actual bag containing the current voter. -fn current_bag_for(id: &AccountIdOf) -> Option { - VoterBagFor::::try_get(id).ok() -} - -/// Data structure providing efficient mostly-accurate selection of the top N voters by stake. -/// -/// It's implemented as a set of linked lists. Each linked list comprises a bag of voters of -/// arbitrary and unbounded length, all having a vote weight within a particular constant range. -/// This structure means that voters can be added and removed in `O(1)` time. -/// -/// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. -/// While the users within any particular bag are sorted in an entirely arbitrary order, the overall -/// stake decreases as successive bags are reached. This means that it is valid to truncate -/// iteration at any desired point; only those voters in the lowest bag (who are known to have -/// relatively little power to affect the outcome) can be excluded. This satisfies both the desire -/// for fairness and the requirement for efficiency. -pub struct VoterList(PhantomData); - -impl VoterList { - /// Remove all data associated with the voter list from storage. - pub fn clear() { - crate::VoterCount::::kill(); - crate::VoterBagFor::::remove_all(None); - crate::VoterBags::::remove_all(None); - crate::VoterNodes::::remove_all(None); - } - - /// Regenerate voter data from the `Nominators` and `Validators` storage items. - /// - /// This is expensive and should only ever be performed during a migration, never during - /// consensus. - /// - /// Returns the number of voters migrated. - pub fn regenerate() -> u32 { - Self::clear(); - - let nominators_iter = Nominators::::iter().map(|(id, _)| Voter::nominator(id)); - let validators_iter = Validators::::iter().map(|(id, _)| Voter::validator(id)); - let weight_of = Pallet::::weight_of_fn(); - - Self::insert_many( - nominators_iter.chain(validators_iter), - weight_of, - ) - } - - /// Decode the length of the voter list. - pub fn decode_len() -> Option { - crate::VoterCount::::try_get().ok().map(|n| n.saturated_into()) - } - - /// Iterate over all nodes in all bags in the voter list. - /// - /// Note that this exhaustively attempts to try all possible bag indices. Full iteration can be - /// expensive; it's recommended to limit the number of items with `.take(n)`. - pub fn iter() -> impl Iterator> { - (0..=BagIdx::MAX).filter_map(|bag_idx| Bag::get(bag_idx)).flat_map(|bag| bag.iter()) - } - - /// Insert a new voter into the appropriate bag in the voter list. - /// - /// If the voter is already present in the list, their type will be updated. - /// That case is cheaper than inserting a new voter. - pub fn insert_as(account_id: &AccountIdOf, voter_type: VoterType) { - // if this is an update operation we can complete this easily and cheaply - if !Node::::update_voter_type_for(account_id, voter_type) { - // otherwise, we need to insert from scratch - let weight_of = Pallet::::weight_of_fn(); - let voter = Voter { id: account_id.clone(), voter_type }; - Self::insert(voter, weight_of); - } - } - - /// Insert a new voter into the appropriate bag in the voter list. - pub fn insert(voter: VoterOf, weight_of: impl Fn(&T::AccountId) -> VoteWeight) { - Self::insert_many(sp_std::iter::once(voter), weight_of); - } - - /// Insert several voters into the appropriate bags in the voter list. - /// - /// This is more efficient than repeated calls to `Self::insert`. - pub fn insert_many( - voters: impl IntoIterator>, - weight_of: impl Fn(&T::AccountId) -> VoteWeight, - ) -> u32 { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for voter in voters.into_iter() { - let weight = weight_of(&voter.id); - let bag = notional_bag_for(weight); - bags.entry(bag).or_insert_with(|| Bag::::get_or_make(bag)).insert(voter); - count += 1; - } - - for (_, bag) in bags { - bag.put(); - } - - crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_add(count)); - count - } - - /// Remove a voter (by id) from the voter list. - pub fn remove(voter: &AccountIdOf) { - Self::remove_many(sp_std::iter::once(voter)) - } - - /// Remove many voters (by id) from the voter list. - /// - /// This is more efficient than repeated calls to `Self::remove`. - pub fn remove_many<'a>(voters: impl IntoIterator>) { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for voter_id in voters.into_iter() { - let node = match Node::::from_id(voter_id) { - Some(node) => node, - None => continue, - }; - count += 1; - - // clear the bag head/tail pointers as necessary - let bag = bags.entry(node.bag_idx).or_insert_with(|| Bag::::get_or_make(node.bag_idx)); - bag.remove_node(&node); - - // now get rid of the node itself - crate::VoterNodes::::remove(node.bag_idx, voter_id); - crate::VoterBagFor::::remove(voter_id); - } - - for (_, bag) in bags { - bag.put(); - } - - crate::VoterCount::::mutate(|prev_count| *prev_count = prev_count.saturating_sub(count)); - } - - /// Update a voter's position in the voter list. - /// - /// If the voter was in the correct bag, no effect. If the voter was in the incorrect bag, they - /// are moved into the correct bag. - /// - /// Returns `true` if the voter moved. - /// - /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by - /// [`self.insert`]. However, given large quantities of voters to move, it may be more efficient - /// to call [`self.remove_many`] followed by [`self.insert_many`]. - pub fn update_position_for( - mut node: Node, - weight_of: impl Fn(&AccountIdOf) -> VoteWeight, - ) -> Option<(BagIdx, BagIdx)> { - node.is_misplaced(&weight_of).then(move || { - let old_idx = node.bag_idx; - - // clear the old bag head/tail pointers as necessary - if let Some(mut bag) = Bag::::get(node.bag_idx) { - bag.remove_node(&node); - bag.put(); - } else { - debug_assert!(false, "every node must have an extant bag associated with it"); - crate::log!( - error, - "Node for staker {:?} did not have a bag; VoterBags is in an inconsistent state", - node.voter.id, - ); - } - - // put the voter into the appropriate new bag - let new_idx = notional_bag_for(weight_of(&node.voter.id)); - node.bag_idx = new_idx; - let mut bag = Bag::::get_or_make(node.bag_idx); - bag.insert_node(node); - bag.put(); - - (old_idx, new_idx) - }) - } -} - -/// A Bag is a doubly-linked list of voters. -/// -/// Note that we maintain both head and tail pointers. While it would be possible to get away -/// with maintaining only a head pointer and cons-ing elements onto the front of the list, it's -/// more desirable to ensure that there is some element of first-come, first-serve to the list's -/// iteration so that there's no incentive to churn voter positioning to improve the chances of -/// appearing within the voter set. -#[derive(DefaultNoBound, Encode, Decode)] -pub struct Bag { - head: Option>, - tail: Option>, - - #[codec(skip)] - bag_idx: BagIdx, -} - -impl Bag { - /// Get a bag by idx. - pub fn get(bag_idx: BagIdx) -> Option> { - crate::VoterBags::::try_get(bag_idx).ok().map(|mut bag| { - bag.bag_idx = bag_idx; - bag - }) - } - - /// Get a bag by idx or make it, appropriately initialized. - pub fn get_or_make(bag_idx: BagIdx) -> Bag { - Self::get(bag_idx).unwrap_or(Bag { bag_idx, ..Default::default() }) - } - - /// Put the bag back into storage. - pub fn put(self) { - crate::VoterBags::::insert(self.bag_idx, self); - } - - /// Get the head node in this bag. - pub fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(self.bag_idx, id)) - } - - /// Get the tail node in this bag. - pub fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(self.bag_idx, id)) - } - - /// Iterate over the nodes in this bag. - pub fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) - } - - /// Insert a new voter into this bag. - /// - /// This is private on purpose because it's naive: it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the nodes. You still need to call - /// `self.put()` after use. - fn insert(&mut self, voter: VoterOf) { - self.insert_node(Node:: { - voter, - prev: None, - next: None, - bag_idx: self.bag_idx, - }); - } - - /// Insert a voter node into this bag. - /// - /// This is private on purpose because it's naive; it doesn't check whether this is the - /// appropriate bag for this voter at all. Generally, use [`VoterList::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the node. You still need to call - /// `self.put()` after use. - fn insert_node(&mut self, mut node: Node) { - let id = node.voter.id.clone(); - - node.prev = self.tail.clone(); - node.next = None; - node.put(); - - // update the previous tail - if let Some(mut tail) = self.tail() { - tail.next = Some(id.clone()); - tail.put(); - } - - // update the internal bag links - if self.head.is_none() { - self.head = Some(id.clone()); - } - self.tail = Some(id.clone()); - - crate::VoterBagFor::::insert(id, self.bag_idx); - } - - /// Remove a voter node from this bag. - /// - /// This is private on purpose because it doesn't check whether this bag contains the voter in - /// the first place. Generally, use [`VoterList::remove`] instead. - /// - /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()` and `node.put()` after use. - fn remove_node(&mut self, node: &Node) { - node.excise(); - - // clear the bag head/tail pointers as necessary - if self.head.as_ref() == Some(&node.voter.id) { - self.head = node.next.clone(); - } - if self.tail.as_ref() == Some(&node.voter.id) { - self.tail = node.prev.clone(); - } - } -} - -/// A Node is the fundamental element comprising the doubly-linked lists which for each bag. -#[derive(Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct Node { - voter: Voter>, - prev: Option>, - next: Option>, - - /// The bag index is not stored in storage, but injected during all fetch operations. - #[codec(skip)] - pub(crate) bag_idx: BagIdx, -} - -impl Node { - /// Get a node by bag idx and account id. - pub fn get(bag_idx: BagIdx, account_id: &AccountIdOf) -> Option> { - crate::VoterNodes::::try_get(&bag_idx, account_id).ok().map(|mut node| { - node.bag_idx = bag_idx; - node - }) - } - - /// Get a node by account id. - /// - /// Note that this must perform two storage lookups: one to identify which bag is appropriate, - /// and another to actually fetch the node. - pub fn from_id(account_id: &AccountIdOf) -> Option> { - let bag = current_bag_for::(account_id)?; - Self::get(bag, account_id) - } - - /// Get a node by account id, assuming it's in the same bag as this node. - pub fn in_bag(&self, account_id: &AccountIdOf) -> Option> { - Self::get(self.bag_idx, account_id) - } - - /// Put the node back into storage. - pub fn put(self) { - crate::VoterNodes::::insert(self.bag_idx, self.voter.id.clone(), self); - } - - /// Get the previous node in the bag. - pub fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| self.in_bag(id)) - } - - /// Get the next node in the bag. - pub fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| self.in_bag(id)) - } - - /// Get this voter's voting data. - pub fn voting_data( - &self, - weight_of: impl Fn(&T::AccountId) -> VoteWeight, - slashing_spans: &BTreeMap, SlashingSpans>, - ) -> Option> { - match self.voter.voter_type { - VoterType::Validator => Some(( - self.voter.id.clone(), - weight_of(&self.voter.id), - sp_std::vec![self.voter.id.clone()], - )), - VoterType::Nominator => { - let Nominations { submitted_in, mut targets, .. } = - Nominators::::get(self.voter.id.clone())?; - // Filter out nomination targets which were nominated before the most recent - // slashing span. - targets.retain(|stash| { - slashing_spans - .get(stash) - .map_or(true, |spans| submitted_in >= spans.last_nonzero_slash()) - }); - - (!targets.is_empty()) - .then(move || (self.voter.id.clone(), weight_of(&self.voter.id), targets)) - } - } - } - - /// Remove this node from the linked list. - /// - /// Modifies storage, but only modifies the adjacent nodes. Does not modify `self` or any bag. - fn excise(&self) { - if let Some(mut prev) = self.prev() { - prev.next = self.next.clone(); - prev.put(); - } - if let Some(mut next) = self.next() { - next.prev = self.prev.clone(); - next.put(); - } - } - - /// `true` when this voter is in the wrong bag. - pub fn is_misplaced(&self, weight_of: impl Fn(&T::AccountId) -> VoteWeight) -> bool { - notional_bag_for(weight_of(&self.voter.id)) != self.bag_idx - } - - /// Update the voter type associated with a particular node by id. - /// - /// This updates storage immediately. - /// - /// Returns whether the voter existed and was successfully updated. - pub fn update_voter_type_for(account_id: &AccountIdOf, voter_type: VoterType) -> bool { - let node = Self::from_id(account_id); - let existed = node.is_some(); - if let Some(mut node) = node { - node.voter.voter_type = voter_type; - node.put(); - } - existed - } - - /// Get the index of the bag that this node _should_ be in, given its vote weight. - /// - /// This is a helper intended only for benchmarking and should not be used in production. - #[cfg(any(test, feature = "runtime-benchmarks"))] - pub fn proper_bag_for(&self) -> BagIdx { - let weight_of = crate::Pallet::::weight_of_fn(); - let current_weight = weight_of(&self.voter.id); - notional_bag_for(current_weight) - } -} - -/// Fundamental information about a voter. -#[derive(Clone, Encode, Decode)] -#[cfg_attr(feature = "std", derive(Debug))] -pub struct Voter { - /// Account Id of this voter - pub id: AccountId, - /// Whether the voter is a validator or nominator - pub voter_type: VoterType, -} - -impl Voter { - pub fn nominator(id: AccountId) -> Self { - Self { - id, - voter_type: VoterType::Nominator, - } - } - - pub fn validator(id: AccountId) -> Self { - Self { - id, - voter_type: VoterType::Validator, - } - } -} - -/// Type of voter. -/// -/// Similar to [`crate::StakerStatus`], but somewhat more limited. -#[derive(Clone, Copy, Encode, Decode, PartialEq, Eq)] -#[cfg_attr(feature = "std", derive(Debug))] -pub enum VoterType { - Validator, - Nominator, -} diff --git a/frame/staking/src/voter_bags/thresholds.rs b/frame/staking/src/voter_bags/thresholds.rs deleted file mode 100644 index 710403d75e64f..0000000000000 --- a/frame/staking/src/voter_bags/thresholds.rs +++ /dev/null @@ -1,229 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) 2021 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Generated voter bag thresholds. - -use super::N_BAGS; - -/// Ratio between adjacent bags; -#[cfg(any(test, feature = "std"))] -#[allow(unused)] -pub const CONSTANT_RATIO: f64 = 1.2483305489016119; - -/// Upper thresholds for each bag. -pub const THRESHOLDS: [u64; N_BAGS as usize] = [ - 1, - 2, - 3, - 4, - 5, - 6, - 7, - 8, - 9, - 11, - 13, - 16, - 19, - 23, - 28, - 34, - 42, - 52, - 64, - 79, - 98, - 122, - 152, - 189, - 235, - 293, - 365, - 455, - 567, - 707, - 882, - 1101, - 1374, - 1715, - 2140, - 2671, - 3334, - 4161, - 5194, - 6483, - 8092, - 10101, - 12609, - 15740, - 19648, - 24527, - 30617, - 38220, - 47711, - 59559, - 74349, - 92812, - 115860, - 144631, - 180547, - 225382, - 281351, - 351219, - 438437, - 547314, - 683228, - 852894, - 1064693, - 1329088, - 1659141, - 2071156, - 2585487, - 3227542, - 4029039, - 5029572, - 6278568, - 7837728, - 9784075, - 12213759, - 15246808, - 19033056, - 23759545, - 29659765, - 37025190, - 46219675, - 57697432, - 72025466, - 89911589, - 112239383, - 140111850, - 174905902, - 218340380, - 272560966, - 340246180, - 424739700, - 530215542, - 661884258, - 826250339, - 1031433539, - 1287569995, - 1607312958, - 2006457867, - 2504722650, - 3126721800, - 3903182340, - 4872461752, - 6082442853, - 7592899225, - 9478448057, - 11832236265, - 14770541991, - 18438518791, - 23017366283, - 28733281486, - 35868633049, - 44775910382, - 55895136784, - 69775606782, - 87103021514, - 108733362657, - 135735178289, - 169442369618, - 211520086272, - 264046985399, - 329617918218, - 411472116776, - 513653213392, - 641208997818, - 800440780206, - 999214678517, - 1247350208103, - 1557105369953, - 1943782201171, - 2426482702132, - 3029052483452, - 3781258749319, - 4720260810076, - 5892445768000, - 7355720059940, - 9182370059991, - 11462633057206, - 14309155016159, - 17862555335640, - 22298373506924, - 27835740839511, - 34748205641269, - 43377246621511, - 54149142084871, - 67596028261358, - 84382187063069, - 105336861893959, - 131495222627659, - 164149503440725, - 204912839732087, - 255798957699744, - 319321653273781, - 398618974707429, - 497608243499122, - 621179571745225, - 775437435763184, - 968002239825113, - 1208386767378873, - 1508466116607513, - 1883064335344139, - 2350686735357198, - 2934434062644189, - 3663143684136207, - 4572814165923224, - 5708383617772005, - 7125949654914296, - 8895540644164415, - 11104575135106362, - 13862180373726516, - 17304583234907174, - 21601839888145304, - 26966236644853160, - 33662776992680304, - 42022272880825152, - 52457686971413784, - 65484533171133904, - 81746343238087392, - 102046457525101200, - 127387710335774608, - 159021970366777056, - 198511983555374656, - 247808573395228608, - 309347012448991104, - 386167325851522816, - 482064469848099072, - 601775804251442048, - 751215120036911616, - 937764783138868096, - 1170640426476344320, - 1461346206149632000, - 1824243111658058240, - 2277258404906088192, - 2842771234587226112, - 3548718175673985024, - 4429973308136232448, - 5530071011365192704, - 6903356581082402816, - 8617670910126150656, - 10757701857491230720, - 13429167864681918464, - 18446744073709551615, -];