From 01513b975b9ccee1383d04c769075e8efc8864e9 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 17 Apr 2023 10:42:11 +0530 Subject: [PATCH 01/55] Initial setup --- Cargo.lock | 49 +++++++++++++++++++ bin/node-template/pallets/template/Cargo.toml | 1 + bin/node-template/pallets/template/src/lib.rs | 21 +++++--- .../pallets/template/src/section.rs | 17 +++++++ frame/support/procedural/Cargo.toml | 1 + frame/support/procedural/src/lib.rs | 23 +++++++++ frame/support/src/lib.rs | 4 +- 7 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 bin/node-template/pallets/template/src/section.rs diff --git a/Cargo.lock b/Cargo.lock index bb8081221f0e2..5dca7bc95460a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2486,6 +2486,7 @@ dependencies = [ "derive-syn-parse", "frame-support-procedural-tools", "itertools", + "macro_magic", "proc-macro-warning", "proc-macro2", "quote", @@ -4484,6 +4485,53 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_magic" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "095e666ccc1f8414c3eb162bae48f75ffc8daaf7a6533100f6887dd82ae7fc00" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.14", +] + +[[package]] +name = "macro_magic_core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32a8723ac45b0e53b693731cad558c238922bb5b9714e3509910e8cb74608d4" +dependencies = [ + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.14", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eadf29df043ac09b33b9c14a5eb738b5f3c9c6d1220a1ad54e0ff15eebd0e8af" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.14", +] + +[[package]] +name = "macro_magic_macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1d0f5d071fdf1bcd3fcec09df36025ad430960a51dd6c60293ef1ad04825fff" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.14", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -6860,6 +6908,7 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", + "macro_magic", "parity-scale-codec", "scale-info", "sp-core", diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index bb6d8c511af7e..baa9005ef626f 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -20,6 +20,7 @@ scale-info = { version = "2.5.0", default-features = false, features = ["derive" frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../../../frame/benchmarking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/system" } +macro_magic = { version = "0.3.0", features = ["proc_support"] } [dev-dependencies] sp-core = { version = "7.0.0", path = "../../../../primitives/core" } diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index 9550d3d546cca..67bda283c1db8 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -14,8 +14,15 @@ mod tests; #[cfg(feature = "runtime-benchmarks")] mod benchmarking; pub mod weights; + +mod section; + pub use weights::*; +use macro_magic::*; +use frame_support::pallet_macros::*; + +#[import_section(section)] #[frame_support::pallet] pub mod pallet { use super::*; @@ -34,13 +41,13 @@ pub mod pallet { type WeightInfo: WeightInfo; } - // The pallet's runtime storage items. - // https://docs.substrate.io/main-docs/build/runtime-storage/ - #[pallet::storage] - #[pallet::getter(fn something)] - // Learn more about declaring storage items: - // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items - pub type Something = StorageValue<_, u32>; + // // The pallet's runtime storage items. + // // https://docs.substrate.io/main-docs/build/runtime-storage/ + // #[pallet::storage] + // #[pallet::getter(fn something)] + // // Learn more about declaring storage items: + // // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items + // pub type Something = StorageValue<_, u32>; // Pallets use events to inform users when important changes are made. // https://docs.substrate.io/main-docs/build/events-errors/ diff --git a/bin/node-template/pallets/template/src/section.rs b/bin/node-template/pallets/template/src/section.rs new file mode 100644 index 0000000000000..968ffa47a114f --- /dev/null +++ b/bin/node-template/pallets/template/src/section.rs @@ -0,0 +1,17 @@ +use macro_magic::*; +use frame_support::pallet_prelude::*; +use frame_system::pallet_prelude::*; + +#[export_tokens] +#[dummy_section] +mod section { + #[pallet::storage] + #[pallet::getter(fn something)] + pub type Something = StorageValue<_, u32>; + + #[pallet::storage] + #[pallet::getter(fn something2)] + pub type Something2 = StorageValue<_, u32>; + + pub fn t1() {} +} \ No newline at end of file diff --git a/frame/support/procedural/Cargo.toml b/frame/support/procedural/Cargo.toml index e508730f43f95..87ea0301a409c 100644 --- a/frame/support/procedural/Cargo.toml +++ b/frame/support/procedural/Cargo.toml @@ -24,6 +24,7 @@ quote = "1.0.26" syn = { version = "2.0.14", features = ["full"] } frame-support-procedural-tools = { version = "4.0.0-dev", path = "./tools" } proc-macro-warning = { version = "0.3.0", default-features = false } +macro_magic = { version = "0.3.0", features = ["proc_support"] } [features] default = ["std"] diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index ea997752c3a53..536be12677fec 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -38,7 +38,10 @@ mod tt_macro; use proc_macro::TokenStream; use quote::quote; +use macro_magic::*; +use syn::{parse_macro_input, ItemMod}; use std::{cell::RefCell, str::FromStr}; +use quote::ToTokens; pub(crate) use storage::INHERENT_INSTANCE_NAME; thread_local! { @@ -1438,3 +1441,23 @@ pub fn origin(_: TokenStream, _: TokenStream) -> TokenStream { pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { pallet_macro_stub() } + +#[import_tokens_attr] +#[proc_macro_attribute] +pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { + let foreign_mod = parse_macro_input!(attr as ItemMod); + let mut internal_mod = parse_macro_input!(tokens as ItemMod); + + if let Some(ref mut content) = internal_mod.content { + content.1.extend(foreign_mod.content.unwrap().1); + } + + quote! { + #internal_mod + }.into() +} + +#[proc_macro_attribute] +pub fn dummy_section(_attr: TokenStream, _tokens: TokenStream) -> TokenStream { + quote! {}.into() +} diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 036bd93464b9f..04ffce51b5602 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2834,7 +2834,9 @@ pub mod pallet_macros { disable_frame_system_supertrait_check, error, event, extra_constants, generate_deposit, generate_storage_info, generate_store, genesis_build, genesis_config, getter, hooks, inherent, origin, storage, storage_prefix, storage_version, type_value, unbounded, - validate_unsigned, weight, whitelist_storage, + validate_unsigned, weight, whitelist_storage, + import_section, __import_tokens_attr_import_section_inner, + dummy_section, }; } From 063ed106429ebad8ca0b47802bd3ca71272071a1 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 18 Apr 2023 10:27:18 +0530 Subject: [PATCH 02/55] Updates macro_magic version and refactors accordingly --- Cargo.lock | 17 +++++---- bin/node-template/pallets/template/Cargo.toml | 2 +- bin/node-template/pallets/template/src/lib.rs | 37 ++++++++++--------- .../pallets/template/src/section.rs | 27 ++++++++------ frame/support/Cargo.toml | 1 + frame/support/procedural/Cargo.toml | 2 +- frame/support/procedural/src/lib.rs | 13 ++++--- frame/support/src/lib.rs | 11 ++++-- 8 files changed, 63 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5dca7bc95460a..e836c66581075 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2454,6 +2454,7 @@ dependencies = [ "impl-trait-for-tuples", "k256", "log", + "macro_magic", "once_cell", "parity-scale-codec", "paste", @@ -4487,9 +4488,9 @@ dependencies = [ [[package]] name = "macro_magic" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "095e666ccc1f8414c3eb162bae48f75ffc8daaf7a6533100f6887dd82ae7fc00" +checksum = "25fc0b6681924e4b8879bb0d6b1ca1e1dc899ca9b9b94044ad9f2b83f3fc7cbd" dependencies = [ "macro_magic_core", "macro_magic_macros", @@ -4499,9 +4500,9 @@ dependencies = [ [[package]] name = "macro_magic_core" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32a8723ac45b0e53b693731cad558c238922bb5b9714e3509910e8cb74608d4" +checksum = "22827765558bdea060552589146cae0d48bd65f6fa9a9b1a1171e97d6d2154fc" dependencies = [ "derive-syn-parse", "macro_magic_core_macros", @@ -4512,9 +4513,9 @@ dependencies = [ [[package]] name = "macro_magic_core_macros" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eadf29df043ac09b33b9c14a5eb738b5f3c9c6d1220a1ad54e0ff15eebd0e8af" +checksum = "871c8044d7dc35f3ac2971f5ee1f24274a4e5050a51a1ebeb746c061156fe32f" dependencies = [ "proc-macro2", "quote", @@ -4523,9 +4524,9 @@ dependencies = [ [[package]] name = "macro_magic_macros" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1d0f5d071fdf1bcd3fcec09df36025ad430960a51dd6c60293ef1ad04825fff" +checksum = "a528638a982a6e41df0d198d7d8be56fb28e17d2b1e702472921871591e9ca55" dependencies = [ "macro_magic_core", "quote", diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index baa9005ef626f..43f749ef5b1cb 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -20,7 +20,7 @@ scale-info = { version = "2.5.0", default-features = false, features = ["derive" frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../../../frame/benchmarking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/system" } -macro_magic = { version = "0.3.0", features = ["proc_support"] } +macro_magic = "0.3.1" [dev-dependencies] sp-core = { version = "7.0.0", path = "../../../../primitives/core" } diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index 67bda283c1db8..9c9b4da4c6a0f 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -19,7 +19,6 @@ mod section; pub use weights::*; -use macro_magic::*; use frame_support::pallet_macros::*; #[import_section(section)] @@ -41,23 +40,23 @@ pub mod pallet { type WeightInfo: WeightInfo; } - // // The pallet's runtime storage items. - // // https://docs.substrate.io/main-docs/build/runtime-storage/ - // #[pallet::storage] - // #[pallet::getter(fn something)] - // // Learn more about declaring storage items: - // // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items - // pub type Something = StorageValue<_, u32>; - - // Pallets use events to inform users when important changes are made. - // https://docs.substrate.io/main-docs/build/events-errors/ - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event { - /// Event documentation should end with an array that provides descriptive names for event - /// parameters. [something, who] - SomethingStored { something: u32, who: T::AccountId }, - } + // The pallet's runtime storage items. + // https://docs.substrate.io/main-docs/build/runtime-storage/ + #[pallet::storage] + #[pallet::getter(fn something)] + // Learn more about declaring storage items: + // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items + pub type Something = StorageValue<_, u32>; + + // // Pallets use events to inform users when important changes are made. + // // https://docs.substrate.io/main-docs/build/events-errors/ + // #[pallet::event] + // #[pallet::generate_deposit(pub(super) fn deposit_event)] + // pub enum Event { + // /// Event documentation should end with an array that provides descriptive names for event + // /// parameters. [something, who] + // SomethingStored { something: u32, who: T::AccountId }, + // } // Errors inform users that something went wrong. #[pallet::error] @@ -86,6 +85,8 @@ pub mod pallet { // Update storage. >::put(something); + // >::put(something); + // Emit an event. Self::deposit_event(Event::SomethingStored { something, who }); // Return a successful DispatchResultWithPostInfo diff --git a/bin/node-template/pallets/template/src/section.rs b/bin/node-template/pallets/template/src/section.rs index 968ffa47a114f..223ff28af1185 100644 --- a/bin/node-template/pallets/template/src/section.rs +++ b/bin/node-template/pallets/template/src/section.rs @@ -1,17 +1,22 @@ -use macro_magic::*; -use frame_support::pallet_prelude::*; -use frame_system::pallet_prelude::*; +use frame_support::pallet_macros::*; -#[export_tokens] -#[dummy_section] +#[export_section] mod section { - #[pallet::storage] - #[pallet::getter(fn something)] - pub type Something = StorageValue<_, u32>; - #[pallet::storage] - #[pallet::getter(fn something2)] - pub type Something2 = StorageValue<_, u32>; + // #[pallet::storage] + // #[pallet::getter(fn something2)] + // pub type Something2 = StorageValue<_, u32>; + + // Pallets use events to inform users when important changes are made. + // https://docs.substrate.io/main-docs/build/events-errors/ + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// Event documentation should end with an array that provides descriptive names for event + /// parameters. [something, who] + SomethingStored { something: u32, who: T::AccountId }, + } + pub fn t1() {} } \ No newline at end of file diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index f62609585818a..cfbacc52ffb78 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -28,6 +28,7 @@ sp-inherents = { version = "4.0.0-dev", default-features = false, path = "../../ sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../primitives/staking" } sp-weights = { version = "4.0.0", default-features = false, path = "../../primitives/weights" } tt-call = "1.0.8" +macro_magic = "0.3.1" frame-support-procedural = { version = "4.0.0-dev", default-features = false, path = "./procedural" } paste = "1.0" once_cell = { version = "1", default-features = false, optional = true } diff --git a/frame/support/procedural/Cargo.toml b/frame/support/procedural/Cargo.toml index 87ea0301a409c..0d4b0349c7f9a 100644 --- a/frame/support/procedural/Cargo.toml +++ b/frame/support/procedural/Cargo.toml @@ -24,7 +24,7 @@ quote = "1.0.26" syn = { version = "2.0.14", features = ["full"] } frame-support-procedural-tools = { version = "4.0.0-dev", path = "./tools" } proc-macro-warning = { version = "0.3.0", default-features = false } -macro_magic = { version = "0.3.0", features = ["proc_support"] } +macro_magic = { version = "0.3.1", features = ["proc_support"] } [features] default = ["std"] diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 536be12677fec..10291c3db7d94 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1442,6 +1442,14 @@ pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { pallet_macro_stub() } +#[proc_macro_attribute] +pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { + match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { + Ok(tokens) => tokens.into(), + Err(err) => err.to_compile_error().into(), + } +} + #[import_tokens_attr] #[proc_macro_attribute] pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { @@ -1456,8 +1464,3 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { #internal_mod }.into() } - -#[proc_macro_attribute] -pub fn dummy_section(_attr: TokenStream, _tokens: TokenStream) -> TokenStream { - quote! {}.into() -} diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 04ffce51b5602..2e9943a449354 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -822,6 +822,10 @@ macro_rules! assert_error_encoded_size { #[doc(hidden)] pub use serde::{Deserialize, Serialize}; +#[doc(hidden)] +#[cfg(not(no_std))] +pub use macro_magic; + #[cfg(test)] pub mod tests { use super::*; @@ -2834,10 +2838,11 @@ pub mod pallet_macros { disable_frame_system_supertrait_check, error, event, extra_constants, generate_deposit, generate_storage_info, generate_store, genesis_build, genesis_config, getter, hooks, inherent, origin, storage, storage_prefix, storage_version, type_value, unbounded, - validate_unsigned, weight, whitelist_storage, - import_section, __import_tokens_attr_import_section_inner, - dummy_section, + validate_unsigned, weight, whitelist_storage, }; + #[macro_magic::use_attr] + pub use frame_support_procedural::import_section; + pub use frame_support_procedural::export_section; } // Generate a macro that will enable/disable code based on `std` feature being active. From c1a1342bbb3ff08d8f9d6de8170b3bbf6e49c1d5 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 18 Apr 2023 10:29:42 +0530 Subject: [PATCH 03/55] Removes unwrap from macro --- frame/support/procedural/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 10291c3db7d94..0318449dd7307 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1457,7 +1457,9 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let mut internal_mod = parse_macro_input!(tokens as ItemMod); if let Some(ref mut content) = internal_mod.content { - content.1.extend(foreign_mod.content.unwrap().1); + if let Some(foreign_content) = foreign_mod.content { + content.1.extend(foreign_content.1); + } } quote! { From 6fe645ba2c793e04591c9cd5eea3bd0153281b73 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 18 Apr 2023 16:46:16 +0530 Subject: [PATCH 04/55] Splits into multiple sections --- .../pallets/template/src/calls.rs | 51 +++++++++++++ .../pallets/template/src/errors.rs | 13 ++++ .../template/src/{section.rs => events.rs} | 10 +-- bin/node-template/pallets/template/src/lib.rs | 76 ++----------------- .../pallets/template/src/storages.rs | 12 +++ 5 files changed, 85 insertions(+), 77 deletions(-) create mode 100644 bin/node-template/pallets/template/src/calls.rs create mode 100644 bin/node-template/pallets/template/src/errors.rs rename bin/node-template/pallets/template/src/{section.rs => events.rs} (76%) create mode 100644 bin/node-template/pallets/template/src/storages.rs diff --git a/bin/node-template/pallets/template/src/calls.rs b/bin/node-template/pallets/template/src/calls.rs new file mode 100644 index 0000000000000..83b6da8340efe --- /dev/null +++ b/bin/node-template/pallets/template/src/calls.rs @@ -0,0 +1,51 @@ +use frame_support::pallet_macros::*; + +#[export_section] +mod calls { + // Dispatchable functions allows users to interact with the pallet and invoke state changes. + // These functions materialize as "extrinsics", which are often compared to transactions. + // Dispatchable functions must be annotated with a weight and must return a DispatchResult. + #[pallet::call] + impl Pallet { + /// An example dispatchable that takes a singles value as a parameter, writes the value to + /// storage and emits an event. This function must be dispatched by a signed extrinsic. + #[pallet::call_index(0)] + #[pallet::weight(T::WeightInfo::do_something())] + pub fn do_something(origin: OriginFor, something: u32) -> DispatchResult { + // Check that the extrinsic was signed and get the signer. + // This function will return an error if the extrinsic is not signed. + // https://docs.substrate.io/main-docs/build/origins/ + let who = ensure_signed(origin)?; + + // Update storage. + >::put(something); + + // >::put(something); + + // Emit an event. + Self::deposit_event(Event::SomethingStored { something, who }); + // Return a successful DispatchResultWithPostInfo + Ok(()) + } + + /// An example dispatchable that may throw a custom error. + #[pallet::call_index(1)] + #[pallet::weight(T::WeightInfo::cause_error())] + pub fn cause_error(origin: OriginFor) -> DispatchResult { + let _who = ensure_signed(origin)?; + + // Read a value from storage. + match >::get() { + // Return an error if the value has not been set. + None => return Err(Error::::NoneValue.into()), + Some(old) => { + // Increment the value read from storage; will error in the event of overflow. + let new = old.checked_add(1).ok_or(Error::::StorageOverflow)?; + // Update the value in storage with the incremented result. + >::put(new); + Ok(()) + }, + } + } + } +} \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/errors.rs b/bin/node-template/pallets/template/src/errors.rs new file mode 100644 index 0000000000000..afc41d2a8ec71 --- /dev/null +++ b/bin/node-template/pallets/template/src/errors.rs @@ -0,0 +1,13 @@ +use frame_support::pallet_macros::*; + +#[export_section] +mod errors { + // Errors inform users that something went wrong. + #[pallet::error] + pub enum Error { + /// Error names should be descriptive. + NoneValue, + /// Errors should have helpful documentation associated with them. + StorageOverflow, + } +} \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/section.rs b/bin/node-template/pallets/template/src/events.rs similarity index 76% rename from bin/node-template/pallets/template/src/section.rs rename to bin/node-template/pallets/template/src/events.rs index 223ff28af1185..93da9342a6db8 100644 --- a/bin/node-template/pallets/template/src/section.rs +++ b/bin/node-template/pallets/template/src/events.rs @@ -1,12 +1,7 @@ use frame_support::pallet_macros::*; #[export_section] -mod section { - - // #[pallet::storage] - // #[pallet::getter(fn something2)] - // pub type Something2 = StorageValue<_, u32>; - +mod events { // Pallets use events to inform users when important changes are made. // https://docs.substrate.io/main-docs/build/events-errors/ #[pallet::event] @@ -16,7 +11,4 @@ mod section { /// parameters. [something, who] SomethingStored { something: u32, who: T::AccountId }, } - - - pub fn t1() {} } \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index 9c9b4da4c6a0f..ba830de7ffac5 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -15,13 +15,19 @@ mod tests; mod benchmarking; pub mod weights; -mod section; +mod storages; +mod events; +mod errors; +mod calls; pub use weights::*; use frame_support::pallet_macros::*; -#[import_section(section)] +#[import_section(events)] +#[import_section(errors)] +#[import_section(calls)] +#[import_section(storages)] #[frame_support::pallet] pub mod pallet { use super::*; @@ -47,70 +53,4 @@ pub mod pallet { // Learn more about declaring storage items: // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items pub type Something = StorageValue<_, u32>; - - // // Pallets use events to inform users when important changes are made. - // // https://docs.substrate.io/main-docs/build/events-errors/ - // #[pallet::event] - // #[pallet::generate_deposit(pub(super) fn deposit_event)] - // pub enum Event { - // /// Event documentation should end with an array that provides descriptive names for event - // /// parameters. [something, who] - // SomethingStored { something: u32, who: T::AccountId }, - // } - - // Errors inform users that something went wrong. - #[pallet::error] - pub enum Error { - /// Error names should be descriptive. - NoneValue, - /// Errors should have helpful documentation associated with them. - StorageOverflow, - } - - // Dispatchable functions allows users to interact with the pallet and invoke state changes. - // These functions materialize as "extrinsics", which are often compared to transactions. - // Dispatchable functions must be annotated with a weight and must return a DispatchResult. - #[pallet::call] - impl Pallet { - /// An example dispatchable that takes a singles value as a parameter, writes the value to - /// storage and emits an event. This function must be dispatched by a signed extrinsic. - #[pallet::call_index(0)] - #[pallet::weight(T::WeightInfo::do_something())] - pub fn do_something(origin: OriginFor, something: u32) -> DispatchResult { - // Check that the extrinsic was signed and get the signer. - // This function will return an error if the extrinsic is not signed. - // https://docs.substrate.io/main-docs/build/origins/ - let who = ensure_signed(origin)?; - - // Update storage. - >::put(something); - - // >::put(something); - - // Emit an event. - Self::deposit_event(Event::SomethingStored { something, who }); - // Return a successful DispatchResultWithPostInfo - Ok(()) - } - - /// An example dispatchable that may throw a custom error. - #[pallet::call_index(1)] - #[pallet::weight(T::WeightInfo::cause_error())] - pub fn cause_error(origin: OriginFor) -> DispatchResult { - let _who = ensure_signed(origin)?; - - // Read a value from storage. - match >::get() { - // Return an error if the value has not been set. - None => return Err(Error::::NoneValue.into()), - Some(old) => { - // Increment the value read from storage; will error in the event of overflow. - let new = old.checked_add(1).ok_or(Error::::StorageOverflow)?; - // Update the value in storage with the incremented result. - >::put(new); - Ok(()) - }, - } - } - } } diff --git a/bin/node-template/pallets/template/src/storages.rs b/bin/node-template/pallets/template/src/storages.rs new file mode 100644 index 0000000000000..979d69c34905f --- /dev/null +++ b/bin/node-template/pallets/template/src/storages.rs @@ -0,0 +1,12 @@ +use frame_support::pallet_macros::*; + +#[export_section] +mod storages { + + // #[pallet::storage] + // #[pallet::getter(fn something2)] + // pub type Something2 = StorageValue<_, u32>; + + + pub fn t1() {} +} \ No newline at end of file From e403600230054178ea4fb3135e1e1affe6f58275 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 19 Apr 2023 10:47:13 +0530 Subject: [PATCH 05/55] Uses call_site to fix macro hygiene issue --- bin/node-template/pallets/template/src/calls.rs | 2 +- bin/node-template/pallets/template/src/storages.rs | 9 +++------ bin/node-template/pallets/template/src/tests.rs | 2 ++ frame/support/procedural/src/pallet/expand/storage.rs | 4 +++- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/node-template/pallets/template/src/calls.rs b/bin/node-template/pallets/template/src/calls.rs index 83b6da8340efe..5af4c9e6dea92 100644 --- a/bin/node-template/pallets/template/src/calls.rs +++ b/bin/node-template/pallets/template/src/calls.rs @@ -20,7 +20,7 @@ mod calls { // Update storage. >::put(something); - // >::put(something); + >::put(something); // Emit an event. Self::deposit_event(Event::SomethingStored { something, who }); diff --git a/bin/node-template/pallets/template/src/storages.rs b/bin/node-template/pallets/template/src/storages.rs index 979d69c34905f..b5058f3d40e38 100644 --- a/bin/node-template/pallets/template/src/storages.rs +++ b/bin/node-template/pallets/template/src/storages.rs @@ -3,10 +3,7 @@ use frame_support::pallet_macros::*; #[export_section] mod storages { - // #[pallet::storage] - // #[pallet::getter(fn something2)] - // pub type Something2 = StorageValue<_, u32>; - - - pub fn t1() {} + #[pallet::storage] + #[pallet::getter(fn something2)] + pub type Something2 = StorageValue<_, u32>; } \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/tests.rs b/bin/node-template/pallets/template/src/tests.rs index 7c2b853ee4dc5..052b934ead10a 100644 --- a/bin/node-template/pallets/template/src/tests.rs +++ b/bin/node-template/pallets/template/src/tests.rs @@ -10,6 +10,8 @@ fn it_works_for_default_value() { assert_ok!(TemplateModule::do_something(RuntimeOrigin::signed(1), 42)); // Read pallet storage and assert an expected result. assert_eq!(TemplateModule::something(), Some(42)); + + assert_eq!(TemplateModule::something2(), Some(42)); // Assert that the correct event was deposited System::assert_last_event(Event::SomethingStored { something: 42, who: 1 }.into()); }); diff --git a/frame/support/procedural/src/pallet/expand/storage.rs b/frame/support/procedural/src/pallet/expand/storage.rs index c742ddcd25fbc..8b7cd4d69b297 100644 --- a/frame/support/procedural/src/pallet/expand/storage.rs +++ b/frame/support/procedural/src/pallet/expand/storage.rs @@ -25,6 +25,7 @@ use crate::{ use quote::ToTokens; use std::{collections::HashMap, ops::IndexMut}; use syn::spanned::Spanned; +use proc_macro2::Span; /// Generate the prefix_ident related to the storage. /// prefix_ident is used for the prefix struct to be given to storage as first generic param. @@ -348,7 +349,8 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { let cfg_attrs = &storage.cfg_attrs; - quote::quote_spanned!(storage.attr_span => + // Since we are using `entries` defined later, we need to specify `call_site` to disable macro hygiene. + quote::quote_spanned!(Span::call_site() => #(#cfg_attrs)* { <#full_ident as #frame_support::storage::StorageEntryMetadataBuilder>::build_metadata( From 666f5486309a0f11307192c346cefc13808c870a Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Sat, 22 Apr 2023 18:15:11 +0530 Subject: [PATCH 06/55] Initial setup --- Cargo.lock | 1 + frame/bags-list/Cargo.toml | 1 + frame/bags-list/src/benchmarks.rs | 3 ++ frame/bags-list/src/lib.rs | 55 +++++++++-------------- frame/bags-list/src/list/mod.rs | 74 +++++++++++++++++++++++++------ frame/bags-list/src/list/tests.rs | 40 +++++------------ frame/bags-list/src/migrations.rs | 18 -------- frame/bags-list/src/tests.rs | 2 +- frame/support/src/lib.rs | 6 +-- 9 files changed, 100 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95d43a76a6188..1f1c789c306e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5745,6 +5745,7 @@ dependencies = [ "frame-support", "frame-system", "log", + "macro_magic", "pallet-balances", "parity-scale-codec", "scale-info", diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 1678ce1ba2ac6..1219336a7b2bb 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -27,6 +27,7 @@ frame-election-provider-support = { version = "4.0.0-dev", default-features = fa # third party log = { version = "0.4.17", default-features = false } +macro_magic = "0.3.1" # Optional imports for benchmarking frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking", optional = true, default-features = false } diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index 0c3955c0d7b79..cda227bd897cd 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -27,6 +27,7 @@ use frame_support::{assert_ok, traits::Get}; use frame_system::RawOrigin as SystemOrigin; use sp_runtime::traits::One; +/* benchmarks_instance_pallet! { rebag_non_terminal { // An expensive case for rebag-ing (rebag a non-terminal node): @@ -194,3 +195,5 @@ benchmarks_instance_pallet! { crate::mock::Runtime ); } + +*/ \ No newline at end of file diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 87eb2d1b341aa..3a96c6fd15f22 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -92,6 +92,9 @@ macro_rules! log { type AccountIdLookupOf = <::Lookup as StaticLookup>::Source; +use frame_support::pallet_macros::*; + +#[import_section(bags_list)] #[frame_support::pallet] pub mod pallet { use super::*; @@ -175,20 +178,6 @@ pub mod pallet { + MaxEncodedLen; } - /// A single node, within some bag. - /// - /// Nodes store links forward and back within their respective bags. - #[pallet::storage] - pub(crate) type ListNodes, I: 'static = ()> = - CountedStorageMap<_, Twox64Concat, T::AccountId, list::Node>; - - /// A bag stored in storage. - /// - /// Stores a `Bag` struct, which stores head and tail pointers to itself. - #[pallet::storage] - pub(crate) type ListBags, I: 'static = ()> = - StorageMap<_, Twox64Concat, T::Score, list::Bag>; - #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event, I: 'static = ()> { @@ -256,21 +245,21 @@ pub mod pallet { } } - #[pallet::hooks] - impl, I: 'static> Hooks> for Pallet { - fn integrity_test() { - // ensure they are strictly increasing, this also implies that duplicates are detected. - assert!( - T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "thresholds must strictly increase, and have no duplicates", - ); - } - - #[cfg(feature = "try-runtime")] - fn try_state(_: BlockNumberFor) -> Result<(), &'static str> { - >::try_state() - } - } + // #[pallet::hooks] + // impl, I: 'static> Hooks> for Pallet { + // fn integrity_test() { + // // ensure they are strictly increasing, this also implies that duplicates are detected. + // assert!( + // T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), + // "thresholds must strictly increase, and have no duplicates", + // ); + // } + + // #[cfg(feature = "try-runtime")] + // fn try_state(_: BlockNumberFor) -> Result<(), &'static str> { + // >::try_state() + // } + // } } #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] @@ -298,11 +287,7 @@ impl, I: 'static> Pallet { Ok(maybe_movement) } - /// Equivalent to `ListBags::get`, but public. Useful for tests in outside of this crate. - #[cfg(feature = "std")] - pub fn list_bags_get(score: T::Score) -> Option> { - ListBags::get(score) - } + } impl, I: 'static> SortedListProvider for Pallet { @@ -408,4 +393,4 @@ impl, I: 'static> ScoreProvider for Pallet { }) } } -} +} \ No newline at end of file diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index f667f4c101ef8..0d59329ed1b60 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -31,6 +31,7 @@ use frame_support::{ defensive, ensure, traits::{Defensive, DefensiveOption, Get}, DefaultNoBound, PalletError, + pallet_macros::*, }; use scale_info::TypeInfo; use sp_runtime::traits::{Bounded, Zero}; @@ -94,8 +95,7 @@ impl, I: 'static> List { /// this function should generally not be used in production as it could lead to a very large /// number of storage accesses. pub(crate) fn unsafe_clear() { - #[allow(deprecated)] - crate::ListBags::::remove_all(None); + crate::list_bags_remove_all::(); #[allow(deprecated)] crate::ListNodes::::remove_all(); } @@ -147,11 +147,11 @@ impl, I: 'static> List { } // we can't check all preconditions, but we can check one - debug_assert!( - crate::ListBags::::iter() - .all(|(threshold, _)| old_thresholds.contains(&threshold)), - "not all `bag_upper` currently in storage are members of `old_thresholds`", - ); + // debug_assert!( + // crate::ListBags::::iter() + // .all(|(threshold, _)| old_thresholds.contains(&threshold)), + // "not all `bag_upper` currently in storage are members of `old_thresholds`", + // ); debug_assert!( crate::ListNodes::::iter() .all(|(_, node)| old_thresholds.contains(&node.bag_upper)), @@ -217,7 +217,7 @@ impl, I: 'static> List { !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), "no id should be present in a removed bag", ); - crate::ListBags::::remove(removed_bag); + crate::list_bags_remove::(removed_bag); } num_affected @@ -616,10 +616,7 @@ impl, I: 'static> Bag { /// Get a bag by its upper score. pub(crate) fn get(bag_upper: T::Score) -> Option> { - crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { - bag.bag_upper = bag_upper; - bag - }) + crate::list_bags_try_get::(bag_upper) } /// Get a bag by its upper score or make it, appropriately initialized. Does not check if @@ -636,9 +633,9 @@ impl, I: 'static> Bag { /// Put the bag back into storage. fn put(self) { if self.is_empty() { - crate::ListBags::::remove(self.bag_upper); + crate::list_bags_remove::(self.bag_upper); } else { - crate::ListBags::::insert(self.bag_upper, self); + crate::list_bags_insert::(self.bag_upper, self); } } @@ -918,3 +915,52 @@ impl, I: 'static> Node { Ok(()) } } + +#[export_section] +mod bags_list { + /// A single node, within some bag. + /// + /// Nodes store links forward and back within their respective bags. + #[pallet::storage] + pub(crate) type ListNodes, I: 'static = ()> = + CountedStorageMap<_, Twox64Concat, T::AccountId, list::Node>; + + /// A bag stored in storage. + /// + /// Stores a `Bag` struct, which stores head and tail pointers to itself. + #[pallet::storage] + type ListBags, I: 'static = ()> = + StorageMap<_, Twox64Concat, T::Score, list::Bag>; + + /// Equivalent to `ListBags::get`, but public. Useful for tests in outside of this crate. + #[cfg(feature = "std")] + pub fn list_bags_get, I: 'static>(score: T::Score) -> Option> { + ListBags::::get(score) + } + + pub fn list_bags_contains_key, I: 'static>(score: T::Score) -> bool { + ListBags::::contains_key(score) + } + + pub fn list_bags_remove_all, I: 'static>() { + #[allow(deprecated)] + ListBags::::remove_all(None); + } + + pub fn list_bags_remove, I: 'static>(score: T::Score) { + ListBags::::remove(score); + } + + /// Get a bag by its upper score. + pub fn list_bags_try_get, I: 'static>(bag_upper: T::Score) -> Option> { + ListBags::::try_get(bag_upper).ok().map(|mut bag| { + // bag.bag_upper = bag_upper; + bag + }) + } + + pub fn list_bags_insert, I: 'static>(bag_upper: T::Score, bag: Bag) { + ListBags::::insert(bag_upper, bag); + } + +} \ No newline at end of file diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index f5afdc24f2608..734955fe694ff 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -1,24 +1,8 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - use super::*; use crate::{ mock::{test_utils::*, *}, - ListBags, ListNodes, + ListNodes, + list_bags_get, list_bags_contains_key }; use frame_election_provider_support::{SortedListProvider, VoteWeight}; use frame_support::{assert_ok, assert_storage_noop}; @@ -37,17 +21,17 @@ fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { assert_eq!(ListNodes::::count(), 4); assert_eq!(ListNodes::::iter().count(), 4); - assert_eq!(ListBags::::iter().count(), 2); + // assert_eq!(ListBags::::iter().count(), 2); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // the state of the bags is as expected assert_eq!( - ListBags::::get(10).unwrap(), + list_bags_get(10).unwrap(), Bag:: { head: Some(1), tail: Some(1), bag_upper: 0, _phantom: PhantomData } ); assert_eq!( - ListBags::::get(1_000).unwrap(), + list_bags_get(1_000).unwrap(), Bag:: { head: Some(2), tail: Some(4), bag_upper: 0, _phantom: PhantomData } ); @@ -156,7 +140,6 @@ fn migrate_works() { mod list { use frame_support::assert_noop; - use super::*; #[test] @@ -254,8 +237,7 @@ mod list { } #[test] - fn remove_works() { - use crate::{ListBags, ListNodes}; + fn remove_works() { let ensure_left = |id, counter| { assert!(!ListNodes::::contains_key(id)); assert_eq!(ListNodes::::count(), counter); @@ -283,8 +265,8 @@ mod list { assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); ensure_left(1, 2); // bag 10 is removed - assert!(!ListBags::::contains_key(10)); - + // assert!(!list_bags_contains_key::::contains_key(10)); + // remove remaining ids to make sure storage cleans up as expected List::::remove(&3).unwrap(); ensure_left(3, 1); @@ -295,7 +277,7 @@ mod list { assert_eq!(get_list_as_ids(), Vec::::new()); // bags are deleted via removals - assert_eq!(ListBags::::iter().count(), 0); + // assert_eq!(ListBags::::iter().count(), 0); }); } @@ -413,7 +395,7 @@ mod list { ListNodes::::insert(11, node_11_no_bag); StakingMock::set_score_of(&10, 14); StakingMock::set_score_of(&11, 15); - assert!(!ListBags::::contains_key(15)); + assert!(!list_bags_contains_key::(15)); assert_eq!(List::::get_bags(), vec![]); // then .. this panics @@ -572,7 +554,7 @@ mod bags { .filter(|bag_upper| !vec![10, 1_000].contains(bag_upper)) .for_each(|bag_upper| { assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); - assert!(!ListBags::::contains_key(*bag_upper)); + assert!(!list_bags_contains_key::(*bag_upper)); }); // when we make a pre-existing bag empty diff --git a/frame/bags-list/src/migrations.rs b/frame/bags-list/src/migrations.rs index 5f9bb8f73ac60..8327bee003477 100644 --- a/frame/bags-list/src/migrations.rs +++ b/frame/bags-list/src/migrations.rs @@ -1,21 +1,3 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! The migrations of this pallet. use codec::{Decode, Encode}; use core::marker::PhantomData; diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 74f1491835a32..d0ec6b856db9f 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -660,4 +660,4 @@ mod sorted_list_provider { assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); }) } -} +} \ No newline at end of file diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 2e9943a449354..9518147e47f9f 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -822,9 +822,9 @@ macro_rules! assert_error_encoded_size { #[doc(hidden)] pub use serde::{Deserialize, Serialize}; -#[doc(hidden)] -#[cfg(not(no_std))] -pub use macro_magic; +// #[doc(hidden)] +// #[cfg(not(no_std))] +// pub use macro_magic; #[cfg(test)] pub mod tests { From 7bee41aea2ee0bc6711528a2732b3c1efdc05b2f Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 24 Apr 2023 09:41:01 +0530 Subject: [PATCH 07/55] Removes unnecessary changes --- frame/bags-list/src/list/mod.rs | 14 ++++++-------- frame/bags-list/src/list/tests.rs | 17 +++++++++++++++++ frame/bags-list/src/migrations.rs | 18 ++++++++++++++++++ frame/bags-list/src/tests.rs | 2 +- frame/support/src/lib.rs | 6 +++--- 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 0d59329ed1b60..78102070a6c5f 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -599,7 +599,7 @@ pub struct Bag, I: 'static = ()> { tail: Option, #[codec(skip)] - bag_upper: T::Score, + pub(super) bag_upper: T::Score, #[codec(skip)] _phantom: PhantomData, } @@ -932,8 +932,6 @@ mod bags_list { type ListBags, I: 'static = ()> = StorageMap<_, Twox64Concat, T::Score, list::Bag>; - /// Equivalent to `ListBags::get`, but public. Useful for tests in outside of this crate. - #[cfg(feature = "std")] pub fn list_bags_get, I: 'static>(score: T::Score) -> Option> { ListBags::::get(score) } @@ -942,24 +940,24 @@ mod bags_list { ListBags::::contains_key(score) } - pub fn list_bags_remove_all, I: 'static>() { + pub(super) fn list_bags_remove_all, I: 'static>() { #[allow(deprecated)] ListBags::::remove_all(None); } - pub fn list_bags_remove, I: 'static>(score: T::Score) { + pub(super) fn list_bags_remove, I: 'static>(score: T::Score) { ListBags::::remove(score); } /// Get a bag by its upper score. - pub fn list_bags_try_get, I: 'static>(bag_upper: T::Score) -> Option> { + pub(super) fn list_bags_try_get, I: 'static>(bag_upper: T::Score) -> Option> { ListBags::::try_get(bag_upper).ok().map(|mut bag| { - // bag.bag_upper = bag_upper; + bag.bag_upper = bag_upper; bag }) } - pub fn list_bags_insert, I: 'static>(bag_upper: T::Score, bag: Bag) { + pub(super) fn list_bags_insert, I: 'static>(bag_upper: T::Score, bag: Bag) { ListBags::::insert(bag_upper, bag); } diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 734955fe694ff..6629445c312ed 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use super::*; use crate::{ mock::{test_utils::*, *}, diff --git a/frame/bags-list/src/migrations.rs b/frame/bags-list/src/migrations.rs index 8327bee003477..5f9bb8f73ac60 100644 --- a/frame/bags-list/src/migrations.rs +++ b/frame/bags-list/src/migrations.rs @@ -1,3 +1,21 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! The migrations of this pallet. use codec::{Decode, Encode}; use core::marker::PhantomData; diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index d0ec6b856db9f..74f1491835a32 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -660,4 +660,4 @@ mod sorted_list_provider { assert!(non_existent_ids.iter().all(|id| !BagsList::contains(id))); }) } -} \ No newline at end of file +} diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 9518147e47f9f..2e9943a449354 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -822,9 +822,9 @@ macro_rules! assert_error_encoded_size { #[doc(hidden)] pub use serde::{Deserialize, Serialize}; -// #[doc(hidden)] -// #[cfg(not(no_std))] -// pub use macro_magic; +#[doc(hidden)] +#[cfg(not(no_std))] +pub use macro_magic; #[cfg(test)] pub mod tests { From 2f9a178fa1ab25c60b6d5cec69f4508a9b2ef6ad Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 24 Apr 2023 09:43:13 +0530 Subject: [PATCH 08/55] Moves template palet back --- bin/node-template/pallets/template/Cargo.toml | 1 - .../pallets/template/src/calls.rs | 51 ------------- .../pallets/template/src/errors.rs | 13 ---- .../pallets/template/src/events.rs | 14 ---- bin/node-template/pallets/template/src/lib.rs | 76 ++++++++++++++++--- .../pallets/template/src/storages.rs | 9 --- .../pallets/template/src/tests.rs | 2 - 7 files changed, 64 insertions(+), 102 deletions(-) delete mode 100644 bin/node-template/pallets/template/src/calls.rs delete mode 100644 bin/node-template/pallets/template/src/errors.rs delete mode 100644 bin/node-template/pallets/template/src/events.rs delete mode 100644 bin/node-template/pallets/template/src/storages.rs diff --git a/bin/node-template/pallets/template/Cargo.toml b/bin/node-template/pallets/template/Cargo.toml index 43f749ef5b1cb..bb6d8c511af7e 100644 --- a/bin/node-template/pallets/template/Cargo.toml +++ b/bin/node-template/pallets/template/Cargo.toml @@ -20,7 +20,6 @@ scale-info = { version = "2.5.0", default-features = false, features = ["derive" frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../../../frame/benchmarking" } frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../../../frame/system" } -macro_magic = "0.3.1" [dev-dependencies] sp-core = { version = "7.0.0", path = "../../../../primitives/core" } diff --git a/bin/node-template/pallets/template/src/calls.rs b/bin/node-template/pallets/template/src/calls.rs deleted file mode 100644 index 5af4c9e6dea92..0000000000000 --- a/bin/node-template/pallets/template/src/calls.rs +++ /dev/null @@ -1,51 +0,0 @@ -use frame_support::pallet_macros::*; - -#[export_section] -mod calls { - // Dispatchable functions allows users to interact with the pallet and invoke state changes. - // These functions materialize as "extrinsics", which are often compared to transactions. - // Dispatchable functions must be annotated with a weight and must return a DispatchResult. - #[pallet::call] - impl Pallet { - /// An example dispatchable that takes a singles value as a parameter, writes the value to - /// storage and emits an event. This function must be dispatched by a signed extrinsic. - #[pallet::call_index(0)] - #[pallet::weight(T::WeightInfo::do_something())] - pub fn do_something(origin: OriginFor, something: u32) -> DispatchResult { - // Check that the extrinsic was signed and get the signer. - // This function will return an error if the extrinsic is not signed. - // https://docs.substrate.io/main-docs/build/origins/ - let who = ensure_signed(origin)?; - - // Update storage. - >::put(something); - - >::put(something); - - // Emit an event. - Self::deposit_event(Event::SomethingStored { something, who }); - // Return a successful DispatchResultWithPostInfo - Ok(()) - } - - /// An example dispatchable that may throw a custom error. - #[pallet::call_index(1)] - #[pallet::weight(T::WeightInfo::cause_error())] - pub fn cause_error(origin: OriginFor) -> DispatchResult { - let _who = ensure_signed(origin)?; - - // Read a value from storage. - match >::get() { - // Return an error if the value has not been set. - None => return Err(Error::::NoneValue.into()), - Some(old) => { - // Increment the value read from storage; will error in the event of overflow. - let new = old.checked_add(1).ok_or(Error::::StorageOverflow)?; - // Update the value in storage with the incremented result. - >::put(new); - Ok(()) - }, - } - } - } -} \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/errors.rs b/bin/node-template/pallets/template/src/errors.rs deleted file mode 100644 index afc41d2a8ec71..0000000000000 --- a/bin/node-template/pallets/template/src/errors.rs +++ /dev/null @@ -1,13 +0,0 @@ -use frame_support::pallet_macros::*; - -#[export_section] -mod errors { - // Errors inform users that something went wrong. - #[pallet::error] - pub enum Error { - /// Error names should be descriptive. - NoneValue, - /// Errors should have helpful documentation associated with them. - StorageOverflow, - } -} \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/events.rs b/bin/node-template/pallets/template/src/events.rs deleted file mode 100644 index 93da9342a6db8..0000000000000 --- a/bin/node-template/pallets/template/src/events.rs +++ /dev/null @@ -1,14 +0,0 @@ -use frame_support::pallet_macros::*; - -#[export_section] -mod events { - // Pallets use events to inform users when important changes are made. - // https://docs.substrate.io/main-docs/build/events-errors/ - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event { - /// Event documentation should end with an array that provides descriptive names for event - /// parameters. [something, who] - SomethingStored { something: u32, who: T::AccountId }, - } -} \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/lib.rs b/bin/node-template/pallets/template/src/lib.rs index ba830de7ffac5..9550d3d546cca 100644 --- a/bin/node-template/pallets/template/src/lib.rs +++ b/bin/node-template/pallets/template/src/lib.rs @@ -14,20 +14,8 @@ mod tests; #[cfg(feature = "runtime-benchmarks")] mod benchmarking; pub mod weights; - -mod storages; -mod events; -mod errors; -mod calls; - pub use weights::*; -use frame_support::pallet_macros::*; - -#[import_section(events)] -#[import_section(errors)] -#[import_section(calls)] -#[import_section(storages)] #[frame_support::pallet] pub mod pallet { use super::*; @@ -53,4 +41,68 @@ pub mod pallet { // Learn more about declaring storage items: // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items pub type Something = StorageValue<_, u32>; + + // Pallets use events to inform users when important changes are made. + // https://docs.substrate.io/main-docs/build/events-errors/ + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// Event documentation should end with an array that provides descriptive names for event + /// parameters. [something, who] + SomethingStored { something: u32, who: T::AccountId }, + } + + // Errors inform users that something went wrong. + #[pallet::error] + pub enum Error { + /// Error names should be descriptive. + NoneValue, + /// Errors should have helpful documentation associated with them. + StorageOverflow, + } + + // Dispatchable functions allows users to interact with the pallet and invoke state changes. + // These functions materialize as "extrinsics", which are often compared to transactions. + // Dispatchable functions must be annotated with a weight and must return a DispatchResult. + #[pallet::call] + impl Pallet { + /// An example dispatchable that takes a singles value as a parameter, writes the value to + /// storage and emits an event. This function must be dispatched by a signed extrinsic. + #[pallet::call_index(0)] + #[pallet::weight(T::WeightInfo::do_something())] + pub fn do_something(origin: OriginFor, something: u32) -> DispatchResult { + // Check that the extrinsic was signed and get the signer. + // This function will return an error if the extrinsic is not signed. + // https://docs.substrate.io/main-docs/build/origins/ + let who = ensure_signed(origin)?; + + // Update storage. + >::put(something); + + // Emit an event. + Self::deposit_event(Event::SomethingStored { something, who }); + // Return a successful DispatchResultWithPostInfo + Ok(()) + } + + /// An example dispatchable that may throw a custom error. + #[pallet::call_index(1)] + #[pallet::weight(T::WeightInfo::cause_error())] + pub fn cause_error(origin: OriginFor) -> DispatchResult { + let _who = ensure_signed(origin)?; + + // Read a value from storage. + match >::get() { + // Return an error if the value has not been set. + None => return Err(Error::::NoneValue.into()), + Some(old) => { + // Increment the value read from storage; will error in the event of overflow. + let new = old.checked_add(1).ok_or(Error::::StorageOverflow)?; + // Update the value in storage with the incremented result. + >::put(new); + Ok(()) + }, + } + } + } } diff --git a/bin/node-template/pallets/template/src/storages.rs b/bin/node-template/pallets/template/src/storages.rs deleted file mode 100644 index b5058f3d40e38..0000000000000 --- a/bin/node-template/pallets/template/src/storages.rs +++ /dev/null @@ -1,9 +0,0 @@ -use frame_support::pallet_macros::*; - -#[export_section] -mod storages { - - #[pallet::storage] - #[pallet::getter(fn something2)] - pub type Something2 = StorageValue<_, u32>; -} \ No newline at end of file diff --git a/bin/node-template/pallets/template/src/tests.rs b/bin/node-template/pallets/template/src/tests.rs index 052b934ead10a..7c2b853ee4dc5 100644 --- a/bin/node-template/pallets/template/src/tests.rs +++ b/bin/node-template/pallets/template/src/tests.rs @@ -10,8 +10,6 @@ fn it_works_for_default_value() { assert_ok!(TemplateModule::do_something(RuntimeOrigin::signed(1), 42)); // Read pallet storage and assert an expected result. assert_eq!(TemplateModule::something(), Some(42)); - - assert_eq!(TemplateModule::something2(), Some(42)); // Assert that the correct event was deposited System::assert_last_event(Event::SomethingStored { something: 42, who: 1 }.into()); }); From 6e18ed6ca64703547fada077215a76af811b6599 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 24 Apr 2023 09:46:01 +0530 Subject: [PATCH 09/55] Updates cargo.lock --- Cargo.lock | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1f1c789c306e7..86c7654d85d29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6909,7 +6909,6 @@ dependencies = [ "frame-benchmarking", "frame-support", "frame-system", - "macro_magic", "parity-scale-codec", "scale-info", "sp-core", From edfe22b87da9b1016b51c5ec77763e21bda15416 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 24 Apr 2023 10:27:29 +0530 Subject: [PATCH 10/55] Moves BagsList inside mod --- frame/bags-list/src/benchmarks.rs | 5 +- frame/bags-list/src/lib.rs | 5 +- frame/bags-list/src/list/mod.rs | 1636 ++++++++++++++--------------- frame/bags-list/src/list/tests.rs | 8 +- frame/bags-list/src/mock.rs | 2 +- frame/bags-list/src/tests.rs | 4 +- 6 files changed, 822 insertions(+), 838 deletions(-) diff --git a/frame/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs index cda227bd897cd..2dae68452a9d2 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/bags-list/src/benchmarks.rs @@ -18,7 +18,7 @@ //! Benchmarks for the bags list pallet. use super::*; -use crate::list::List; +use crate::List; use frame_benchmarking::v1::{ account, benchmarks_instance_pallet, whitelist_account, whitelisted_caller, }; @@ -27,7 +27,6 @@ use frame_support::{assert_ok, traits::Get}; use frame_system::RawOrigin as SystemOrigin; use sp_runtime::traits::One; -/* benchmarks_instance_pallet! { rebag_non_terminal { // An expensive case for rebag-ing (rebag a non-terminal node): @@ -195,5 +194,3 @@ benchmarks_instance_pallet! { crate::mock::Runtime ); } - -*/ \ No newline at end of file diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 3a96c6fd15f22..2d0fc5b7ea01b 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -70,7 +70,6 @@ pub mod mock; mod tests; pub mod weights; -pub use list::{notional_bag_for, Bag, List, ListError, Node}; pub use pallet::*; pub use weights::WeightInfo; @@ -278,7 +277,7 @@ impl, I: 'static> Pallet { new_score: T::Score, ) -> Result, ListError> { // If no voter at that node, don't do anything. the caller just wasted the fee to call this. - let node = list::Node::::get(&account).ok_or(ListError::NodeNotFound)?; + let node = Node::::get(&account).ok_or(ListError::NodeNotFound)?; let maybe_movement = List::update_position_for(node, new_score); if let Some((from, to)) = maybe_movement { Self::deposit_event(Event::::Rebagged { who: account.clone(), from, to }); @@ -355,7 +354,7 @@ impl, I: 'static> SortedListProvider for Pallet fn score_update_worst_case(who: &T::AccountId, is_increase: bool) -> Self::Score { use frame_support::traits::Get as _; let thresholds = T::BagThresholds::get(); - let node = list::Node::::get(who).unwrap(); + let node = Node::::get(who).unwrap(); let current_bag_idx = thresholds .iter() .chain(sp_std::iter::once(&T::Score::max_value())) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 78102070a6c5f..44599a125b0a9 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -24,915 +24,708 @@ //! of the aggregate linked list. All operations with the bags list should happen through this //! interface. -use crate::Config; -use codec::{Decode, Encode, MaxEncodedLen}; -use frame_election_provider_support::ScoreProvider; -use frame_support::{ - defensive, ensure, - traits::{Defensive, DefensiveOption, Get}, - DefaultNoBound, PalletError, - pallet_macros::*, -}; -use scale_info::TypeInfo; -use sp_runtime::traits::{Bounded, Zero}; -use sp_std::{ - boxed::Box, - collections::{btree_map::BTreeMap, btree_set::BTreeSet}, - iter, - marker::PhantomData, - prelude::*, -}; - -#[derive(Debug, PartialEq, Eq, Encode, Decode, MaxEncodedLen, TypeInfo, PalletError)] -pub enum ListError { - /// A duplicate id has been detected. - Duplicate, - /// An Id does not have a greater score than another Id. - NotHeavier, - /// Attempted to place node in front of a node in another bag. - NotInSameBag, - /// Given node id was not found. - NodeNotFound, -} +use frame_support::pallet_macros::*; #[cfg(test)] mod tests; -/// Given a certain score, to which bag does it belong to? -/// -/// Bags are identified by their upper threshold; the value returned by this function is guaranteed -/// to be a member of `T::BagThresholds`. -/// -/// Note that even if the thresholds list does not have `T::Score::max_value()` as its final member, -/// this function behaves as if it does. -pub fn notional_bag_for, I: 'static>(score: T::Score) -> T::Score { - let thresholds = T::BagThresholds::get(); - let idx = thresholds.partition_point(|&threshold| score > threshold); - thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) -} - -/// The **ONLY** entry point of this module. All operations to the bags-list should happen through -/// this interface. It is forbidden to access other module members directly. -// -// Data structure providing efficient mostly-accurate selection of the top N id by `Score`. -// -// It's implemented as a set of linked lists. Each linked list comprises a bag of ids of -// arbitrary and unbounded length, all having a score within a particular constant range. -// This structure means that ids can be added and removed in `O(1)` time. -// -// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While -// the users within any particular bag are sorted in an entirely arbitrary order, the overall score -// decreases as successive bags are reached. This means that it is valid to truncate -// iteration at any desired point; only those ids in the lowest bag can be excluded. This -// satisfies both the desire for fairness and the requirement for efficiency. -pub struct List, I: 'static = ()>(PhantomData<(T, I)>); - -impl, I: 'static> List { - /// Remove all data associated with the list from storage. - /// - /// ## WARNING - /// - /// this function should generally not be used in production as it could lead to a very large - /// number of storage accesses. - pub(crate) fn unsafe_clear() { - crate::list_bags_remove_all::(); - #[allow(deprecated)] - crate::ListNodes::::remove_all(); - } - - /// Regenerate all of the data from the given ids. - /// - /// WARNING: this is expensive and should only ever be performed when the list needs to be - /// generated from scratch. Care needs to be taken to ensure - /// - /// This may or may not need to be called at genesis as well, based on the configuration of the - /// pallet using this `List`. - /// - /// Returns the number of ids migrated. - pub fn unsafe_regenerate( - all: impl IntoIterator, - score_of: Box T::Score>, - ) -> u32 { - // NOTE: This call is unsafe for the same reason as SortedListProvider::unsafe_regenerate. - // I.e. because it can lead to many storage accesses. - // So it is ok to call it as caller must ensure the conditions. - Self::unsafe_clear(); - Self::insert_many(all, score_of) - } - - /// Migrate the list from one set of thresholds to another. - /// - /// This should only be called as part of an intentional migration; it's fairly expensive. - /// - /// Returns the number of accounts affected. - /// - /// Preconditions: - /// - /// - `old_thresholds` is the previous list of thresholds. - /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::BagThresholds` has already been updated and is the new set of thresholds. - /// - /// Postconditions: - /// - /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. - /// - No id is changed unless required to by the difference between the old threshold list and - /// the new. - /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the new - /// threshold set. - #[allow(dead_code)] - pub fn migrate(old_thresholds: &[T::Score]) -> u32 { - let new_thresholds = T::BagThresholds::get(); - if new_thresholds == old_thresholds { - return 0 - } - - // we can't check all preconditions, but we can check one - // debug_assert!( - // crate::ListBags::::iter() - // .all(|(threshold, _)| old_thresholds.contains(&threshold)), - // "not all `bag_upper` currently in storage are members of `old_thresholds`", - // ); - debug_assert!( - crate::ListNodes::::iter() - .all(|(_, node)| old_thresholds.contains(&node.bag_upper)), - "not all `node.bag_upper` currently in storage are members of `old_thresholds`", - ); - - let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = new_thresholds.iter().copied().collect(); - - // accounts that need to be rebagged - let mut affected_accounts = BTreeSet::new(); - // track affected old bags to make sure we only iterate them once - let mut affected_old_bags = BTreeSet::new(); - - let new_bags = new_set.difference(&old_set).copied(); - // a new bag means that all accounts previously using the old bag's threshold must now - // be rebagged - for inserted_bag in new_bags { - let affected_bag = { - // this recreates `notional_bag_for` logic, but with the old thresholds. - let idx = old_thresholds.partition_point(|&threshold| inserted_bag > threshold); - old_thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) - }; - if !affected_old_bags.insert(affected_bag) { - // If the previous threshold list was [10, 20], and we insert [3, 5], then there's - // no point iterating through bag 10 twice. - continue - } - - if let Some(bag) = Bag::::get(affected_bag) { - affected_accounts.extend(bag.iter().map(|node| node.id)); - } +#[export_section] +mod bags_list { + use codec::{Decode, Encode, MaxEncodedLen}; + use frame_election_provider_support::ScoreProvider; + use frame_support::{ + defensive, ensure, + traits::{Defensive, DefensiveOption, Get}, + DefaultNoBound, PalletError, + }; + use scale_info::TypeInfo; + use sp_runtime::traits::{Bounded, Zero}; + use sp_std::{ + boxed::Box, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + iter, + marker::PhantomData, + prelude::*, + }; + + #[derive(Debug, PartialEq, Eq, Encode, Decode, MaxEncodedLen, TypeInfo, PalletError)] + pub enum ListError { + /// A duplicate id has been detected. + Duplicate, + /// An Id does not have a greater score than another Id. + NotHeavier, + /// Attempted to place node in front of a node in another bag. + NotInSameBag, + /// Given node id was not found. + NodeNotFound, + } + + /// Given a certain score, to which bag does it belong to? + /// + /// Bags are identified by their upper threshold; the value returned by this function is guaranteed + /// to be a member of `T::BagThresholds`. + /// + /// Note that even if the thresholds list does not have `T::Score::max_value()` as its final member, + /// this function behaves as if it does. + pub fn notional_bag_for, I: 'static>(score: T::Score) -> T::Score { + let thresholds = T::BagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| score > threshold); + thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) + } + + /// The **ONLY** entry point of this module. All operations to the bags-list should happen through + /// this interface. It is forbidden to access other module members directly. + // + // Data structure providing efficient mostly-accurate selection of the top N id by `Score`. + // + // It's implemented as a set of linked lists. Each linked list comprises a bag of ids of + // arbitrary and unbounded length, all having a score within a particular constant range. + // This structure means that ids can be added and removed in `O(1)` time. + // + // Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While + // the users within any particular bag are sorted in an entirely arbitrary order, the overall score + // decreases as successive bags are reached. This means that it is valid to truncate + // iteration at any desired point; only those ids in the lowest bag can be excluded. This + // satisfies both the desire for fairness and the requirement for efficiency. + pub struct List, I: 'static = ()>(PhantomData<(T, I)>); + + impl, I: 'static> List { + /// Remove all data associated with the list from storage. + /// + /// ## WARNING + /// + /// this function should generally not be used in production as it could lead to a very large + /// number of storage accesses. + pub(crate) fn unsafe_clear() { + #[allow(deprecated)] + ListBags::::remove_all(None); + #[allow(deprecated)] + crate::ListNodes::::remove_all(); } - let removed_bags = old_set.difference(&new_set).copied(); - // a removed bag means that all members of that bag must be rebagged - for removed_bag in removed_bags.clone() { - if !affected_old_bags.insert(removed_bag) { - continue - } + /// Regenerate all of the data from the given ids. + /// + /// WARNING: this is expensive and should only ever be performed when the list needs to be + /// generated from scratch. Care needs to be taken to ensure + /// + /// This may or may not need to be called at genesis as well, based on the configuration of the + /// pallet using this `List`. + /// + /// Returns the number of ids migrated. + pub fn unsafe_regenerate( + all: impl IntoIterator, + score_of: Box T::Score>, + ) -> u32 { + // NOTE: This call is unsafe for the same reason as SortedListProvider::unsafe_regenerate. + // I.e. because it can lead to many storage accesses. + // So it is ok to call it as caller must ensure the conditions. + Self::unsafe_clear(); + Self::insert_many(all, score_of) + } - if let Some(bag) = Bag::::get(removed_bag) { - affected_accounts.extend(bag.iter().map(|node| node.id)); + /// Migrate the list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::BagThresholds` has already been updated and is the new set of thresholds. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. + /// - No id is changed unless required to by the difference between the old threshold list and + /// the new. + /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the new + /// threshold set. + #[allow(dead_code)] + pub fn migrate(old_thresholds: &[T::Score]) -> u32 { + let new_thresholds = T::BagThresholds::get(); + if new_thresholds == old_thresholds { + return 0 } - } - // migrate the voters whose bag has changed - let num_affected = affected_accounts.len() as u32; - let score_of = T::ScoreProvider::score; - let _removed = Self::remove_many(&affected_accounts); - debug_assert_eq!(_removed, num_affected); - let _inserted = Self::insert_many(affected_accounts.into_iter(), score_of); - debug_assert_eq!(_inserted, num_affected); - - // we couldn't previously remove the old bags because both insertion and removal assume that - // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid - // of them. - // - // it's pretty cheap to iterate this again, because both sets are in-memory and require no - // lookups. - for removed_bag in removed_bags { + // we can't check all preconditions, but we can check one debug_assert!( - !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), - "no id should be present in a removed bag", + ListBags::::iter() + .all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + debug_assert!( + crate::ListNodes::::iter() + .all(|(_, node)| old_thresholds.contains(&node.bag_upper)), + "not all `node.bag_upper` currently in storage are members of `old_thresholds`", ); - crate::list_bags_remove::(removed_bag); - } - - num_affected - } - - /// Returns `true` if the list contains `id`, otherwise returns `false`. - pub(crate) fn contains(id: &T::AccountId) -> bool { - crate::ListNodes::::contains_key(id) - } - - /// Get the score of the given node, - pub fn get_score(id: &T::AccountId) -> Result { - Node::::get(id).map(|node| node.score()).ok_or(ListError::NodeNotFound) - } - - /// Iterate over all nodes in all bags in the list. - /// - /// Full iteration can be expensive; it's recommended to limit the number of items with - /// `.take(n)`. - pub(crate) fn iter() -> impl Iterator> { - // We need a touch of special handling here: because we permit `T::BagThresholds` to - // omit the final bound, we need to ensure that we explicitly include that threshold in the - // list. - // - // It's important to retain the ability to omit the final bound because it makes tests much - // easier; they can just configure `type BagThresholds = ()`. - let thresholds = T::BagThresholds::get(); - let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == - Some(&T::Score::max_value()) - { - // in the event that they included it, we can just pass the iterator through unchanged. - Box::new(iter.rev()) - } else { - // otherwise, insert it here. - Box::new(iter.chain(iter::once(T::Score::max_value())).rev()) - }; - - iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) - } - /// Same as `iter`, but we start from a specific node. - /// - /// All items after this node are returned, excluding `start` itself. - pub(crate) fn iter_from( - start: &T::AccountId, - ) -> Result>, ListError> { - // We chain two iterators: - // 1. from the given `start` till the end of the bag - // 2. all the bags that come after `start`'s bag. - - let start_node = Node::::get(start).ok_or(ListError::NodeNotFound)?; - let start_node_upper = start_node.bag_upper; - let start_bag = sp_std::iter::successors(start_node.next(), |prev| prev.next()); + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = new_thresholds.iter().copied().collect(); + + // accounts that need to be rebagged + let mut affected_accounts = BTreeSet::new(); + // track affected old bags to make sure we only iterate them once + let mut affected_old_bags = BTreeSet::new(); + + let new_bags = new_set.difference(&old_set).copied(); + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_bags { + let affected_bag = { + // this recreates `notional_bag_for` logic, but with the old thresholds. + let idx = old_thresholds.partition_point(|&threshold| inserted_bag > threshold); + old_thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) + }; + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } + } - let thresholds = T::BagThresholds::get(); - let idx = thresholds.partition_point(|&threshold| start_node_upper > threshold); - let leftover_bags = thresholds - .into_iter() - .take(idx) - .copied() - .rev() - .filter_map(Bag::get) - .flat_map(|bag| bag.iter()); - - Ok(start_bag.chain(leftover_bags)) - } + let removed_bags = old_set.difference(&new_set).copied(); + // a removed bag means that all members of that bag must be rebagged + for removed_bag in removed_bags.clone() { + if !affected_old_bags.insert(removed_bag) { + continue + } - /// Insert several ids into the appropriate bags in the list. Continues with insertions - /// if duplicates are detected. - /// - /// Returns the final count of number of ids inserted. - fn insert_many( - ids: impl IntoIterator, - score_of: impl Fn(&T::AccountId) -> T::Score, - ) -> u32 { - let mut count = 0; - ids.into_iter().for_each(|v| { - let score = score_of(&v); - if Self::insert(v, score).is_ok() { - count += 1; + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } } - }); - count - } + // migrate the voters whose bag has changed + let num_affected = affected_accounts.len() as u32; + let score_of = T::ScoreProvider::score; + let _removed = Self::remove_many(&affected_accounts); + debug_assert_eq!(_removed, num_affected); + let _inserted = Self::insert_many(affected_accounts.into_iter(), score_of); + debug_assert_eq!(_inserted, num_affected); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in removed_bags { + debug_assert!( + !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), + "no id should be present in a removed bag", + ); + ListBags::::remove(removed_bag); + } - /// Insert a new id into the appropriate bag in the list. - /// - /// Returns an error if the list already contains `id`. - pub(crate) fn insert(id: T::AccountId, score: T::Score) -> Result<(), ListError> { - if Self::contains(&id) { - return Err(ListError::Duplicate) - } - - let bag_score = notional_bag_for::(score); - let mut bag = Bag::::get_or_make(bag_score); - // unchecked insertion is okay; we just got the correct `notional_bag_for`. - bag.insert_unchecked(id.clone(), score); - - // new inserts are always the tail, so we must write the bag. - bag.put(); - - crate::log!( - debug, - "inserted {:?} with score {:?} into bag {:?}, new count is {}", - id, - score, - bag_score, - crate::ListNodes::::count(), - ); - - Ok(()) - } + num_affected + } - /// Remove an id from the list, returning an error if `id` does not exists. - pub(crate) fn remove(id: &T::AccountId) -> Result<(), ListError> { - if !Self::contains(id) { - return Err(ListError::NodeNotFound) + /// Returns `true` if the list contains `id`, otherwise returns `false`. + pub(crate) fn contains(id: &T::AccountId) -> bool { + crate::ListNodes::::contains_key(id) } - let _ = Self::remove_many(sp_std::iter::once(id)); - Ok(()) - } - /// Remove many ids from the list. - /// - /// This is more efficient than repeated calls to `Self::remove`. - /// - /// Returns the final count of number of ids removed. - fn remove_many<'a>(ids: impl IntoIterator) -> u32 { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for id in ids.into_iter() { - let node = match Node::::get(id) { - Some(node) => node, - None => continue, - }; - count += 1; + /// Get the score of the given node, + pub fn get_score(id: &T::AccountId) -> Result { + Node::::get(id).map(|node| node.score()).ok_or(ListError::NodeNotFound) + } - if !node.is_terminal() { - // this node is not a head or a tail and thus the bag does not need to be updated - node.excise() + /// Iterate over all nodes in all bags in the list. + /// + /// Full iteration can be expensive; it's recommended to limit the number of items with + /// `.take(n)`. + pub(crate) fn iter() -> impl Iterator> { + // We need a touch of special handling here: because we permit `T::BagThresholds` to + // omit the final bound, we need to ensure that we explicitly include that threshold in the + // list. + // + // It's important to retain the ability to omit the final bound because it makes tests much + // easier; they can just configure `type BagThresholds = ()`. + let thresholds = T::BagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == + Some(&T::Score::max_value()) + { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter.rev()) } else { - // this node is a head or tail, so the bag needs to be updated - let bag = bags - .entry(node.bag_upper) - .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); - // node.bag_upper must be correct, therefore this bag will contain this node. - bag.remove_node_unchecked(&node); - } + // otherwise, insert it here. + Box::new(iter.chain(iter::once(T::Score::max_value())).rev()) + }; - // now get rid of the node itself - node.remove_from_storage_unchecked() + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) } - for (_, bag) in bags { - bag.put(); + /// Same as `iter`, but we start from a specific node. + /// + /// All items after this node are returned, excluding `start` itself. + pub(crate) fn iter_from( + start: &T::AccountId, + ) -> Result>, ListError> { + // We chain two iterators: + // 1. from the given `start` till the end of the bag + // 2. all the bags that come after `start`'s bag. + + let start_node = Node::::get(start).ok_or(ListError::NodeNotFound)?; + let start_node_upper = start_node.bag_upper; + let start_bag = sp_std::iter::successors(start_node.next(), |prev| prev.next()); + + let thresholds = T::BagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| start_node_upper > threshold); + let leftover_bags = thresholds + .into_iter() + .take(idx) + .copied() + .rev() + .filter_map(Bag::get) + .flat_map(|bag| bag.iter()); + + Ok(start_bag.chain(leftover_bags)) } - count - } + /// Insert several ids into the appropriate bags in the list. Continues with insertions + /// if duplicates are detected. + /// + /// Returns the final count of number of ids inserted. + fn insert_many( + ids: impl IntoIterator, + score_of: impl Fn(&T::AccountId) -> T::Score, + ) -> u32 { + let mut count = 0; + ids.into_iter().for_each(|v| { + let score = score_of(&v); + if Self::insert(v, score).is_ok() { + count += 1; + } + }); + + count + } - /// Update a node's position in the list. - /// - /// If the node was in the correct bag, no effect. If the node was in the incorrect bag, they - /// are moved into the correct bag. - /// - /// Returns `Some((old_idx, new_idx))` if the node moved, otherwise `None`. In both cases, the - /// node's score is written to the `score` field. Thus, this is not a noop, even if `None`. - /// - /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by - /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient - /// to call [`self.remove_many`] followed by [`self.insert_many`]. - pub(crate) fn update_position_for( - mut node: Node, - new_score: T::Score, - ) -> Option<(T::Score, T::Score)> { - node.score = new_score; - if node.is_misplaced(new_score) { - let old_bag_upper = node.bag_upper; - - if !node.is_terminal() { - // this node is not a head or a tail, so we can just cut it out of the list. update - // and put the prev and next of this node, we do `node.put` inside `insert_note`. - node.excise(); - } else if let Some(mut bag) = Bag::::get(node.bag_upper) { - // this is a head or tail, so the bag must be updated. - bag.remove_node_unchecked(&node); - bag.put(); - } else { - frame_support::defensive!( - "Node did not have a bag; BagsList is in an inconsistent state" - ); + /// Insert a new id into the appropriate bag in the list. + /// + /// Returns an error if the list already contains `id`. + pub(crate) fn insert(id: T::AccountId, score: T::Score) -> Result<(), ListError> { + if Self::contains(&id) { + return Err(ListError::Duplicate) } - // put the node into the appropriate new bag. - let new_bag_upper = notional_bag_for::(new_score); - let mut bag = Bag::::get_or_make(new_bag_upper); - // prev, next, and bag_upper of the node are updated inside `insert_node`, also - // `node.put` is in there. - bag.insert_node_unchecked(node); + let bag_score = notional_bag_for::(score); + let mut bag = Bag::::get_or_make(bag_score); + // unchecked insertion is okay; we just got the correct `notional_bag_for`. + bag.insert_unchecked(id.clone(), score); + + // new inserts are always the tail, so we must write the bag. bag.put(); - Some((old_bag_upper, new_bag_upper)) - } else { - // just write the new score. - node.put(); - None - } - } + crate::log!( + debug, + "inserted {:?} with score {:?} into bag {:?}, new count is {}", + id, + score, + bag_score, + crate::ListNodes::::count(), + ); - /// Put `heavier_id` to the position directly in front of `lighter_id`. Both ids must be in the - /// same bag and the `score_of` `lighter_id` must be less than that of `heavier_id`. - pub(crate) fn put_in_front_of( - lighter_id: &T::AccountId, - heavier_id: &T::AccountId, - ) -> Result<(), ListError> { - let lighter_node = Node::::get(&lighter_id).ok_or(ListError::NodeNotFound)?; - let heavier_node = Node::::get(&heavier_id).ok_or(ListError::NodeNotFound)?; - - ensure!(lighter_node.bag_upper == heavier_node.bag_upper, ListError::NotInSameBag); - - // this is the most expensive check, so we do it last. - ensure!( - T::ScoreProvider::score(&heavier_id) > T::ScoreProvider::score(&lighter_id), - ListError::NotHeavier - ); - - // remove the heavier node from this list. Note that this removes the node from storage and - // decrements the node counter. - let _ = - Self::remove(&heavier_id).defensive_proof("both nodes have been checked to exist; qed"); - - // re-fetch `lighter_node` from storage since it may have been updated when `heavier_node` - // was removed. - let lighter_node = - Node::::get(lighter_id).defensive_ok_or_else(|| ListError::NodeNotFound)?; - - // insert `heavier_node` directly in front of `lighter_node`. This will update both nodes - // in storage and update the node counter. - Self::insert_at_unchecked(lighter_node, heavier_node); - - Ok(()) - } + Ok(()) + } - /// Insert `node` directly in front of `at`. - /// - /// WARNINGS: - /// - this is a naive function in that it does not check if `node` belongs to the same bag as - /// `at`. It is expected that the call site will check preconditions. - /// - this will panic if `at.bag_upper` is not a bag that already exists in storage. - fn insert_at_unchecked(mut at: Node, mut node: Node) { - // connect `node` to its new `prev`. - node.prev = at.prev.clone(); - if let Some(mut prev) = at.prev() { - prev.next = Some(node.id().clone()); - prev.put() - } - - // connect `node` and `at`. - node.next = Some(at.id().clone()); - at.prev = Some(node.id().clone()); - - if node.is_terminal() { - // `node` is the new head, so we make sure the bag is updated. Note, - // since `node` is always in front of `at` we know that 1) there is always at least 2 - // nodes in the bag, and 2) only `node` could be the head and only `at` could be the - // tail. - let mut bag = Bag::::get(at.bag_upper) - .expect("given nodes must always have a valid bag. qed."); - - if node.prev == None { - bag.head = Some(node.id().clone()) + /// Remove an id from the list, returning an error if `id` does not exists. + pub(crate) fn remove(id: &T::AccountId) -> Result<(), ListError> { + if !Self::contains(id) { + return Err(ListError::NodeNotFound) } + let _ = Self::remove_many(sp_std::iter::once(id)); + Ok(()) + } - bag.put() - }; - - // write the updated nodes to storage. - at.put(); - node.put(); - } - - /// Check the internal state of the list. - /// - /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) - /// is being used, after all other staking data (such as counter) has been updated. It checks: - /// - /// * there are no duplicate ids, - /// * length of this list is in sync with `ListNodes::count()`, - /// * and sanity-checks all bags and nodes. This will cascade down all the checks and makes sure - /// all bags and nodes are checked per *any* update to `List`. - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - pub(crate) fn do_try_state() -> Result<(), &'static str> { - let mut seen_in_list = BTreeSet::new(); - ensure!( - Self::iter().map(|node| node.id).all(|id| seen_in_list.insert(id)), - "duplicate identified", - ); - - let iter_count = Self::iter().count() as u32; - let stored_count = crate::ListNodes::::count(); - let nodes_count = crate::ListNodes::::iter().count() as u32; - ensure!(iter_count == stored_count, "iter_count != stored_count"); - ensure!(stored_count == nodes_count, "stored_count != nodes_count"); - - crate::log!(trace, "count of nodes: {}", stored_count); - - let active_bags = { - let thresholds = T::BagThresholds::get().iter().copied(); - let thresholds: Vec = - if thresholds.clone().last() == Some(T::Score::max_value()) { - // in the event that they included it, we don't need to make any changes - thresholds.collect() - } else { - // otherwise, insert it here. - thresholds.chain(iter::once(T::Score::max_value())).collect() + /// Remove many ids from the list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + /// + /// Returns the final count of number of ids removed. + pub fn remove_many<'a>(ids: impl IntoIterator) -> u32 { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for id in ids.into_iter() { + let node = match Node::::get(id) { + Some(node) => node, + None => continue, }; - thresholds.into_iter().filter_map(|t| Bag::::get(t)) - }; - - let _ = active_bags.clone().try_for_each(|b| b.do_try_state())?; + count += 1; - let nodes_in_bags_count = - active_bags.clone().fold(0u32, |acc, cur| acc + cur.iter().count() as u32); - ensure!(nodes_count == nodes_in_bags_count, "stored_count != nodes_in_bags_count"); + if !node.is_terminal() { + // this node is not a head or a tail and thus the bag does not need to be updated + node.excise() + } else { + // this node is a head or tail, so the bag needs to be updated + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + // node.bag_upper must be correct, therefore this bag will contain this node. + bag.remove_node_unchecked(&node); + } + + // now get rid of the node itself + node.remove_from_storage_unchecked() + } - crate::log!(trace, "count of active bags {}", active_bags.count()); + for (_, bag) in bags { + bag.put(); + } - // check that all nodes are sane. We check the `ListNodes` storage item directly in case we - // have some "stale" nodes that are not in a bag. - for (_id, node) in crate::ListNodes::::iter() { - node.do_try_state()? + count } - Ok(()) - } + /// Update a node's position in the list. + /// + /// If the node was in the correct bag, no effect. If the node was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `Some((old_idx, new_idx))` if the node moved, otherwise `None`. In both cases, the + /// node's score is written to the `score` field. Thus, this is not a noop, even if `None`. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub(crate) fn update_position_for( + mut node: Node, + new_score: T::Score, + ) -> Option<(T::Score, T::Score)> { + node.score = new_score; + if node.is_misplaced(new_score) { + let old_bag_upper = node.bag_upper; + + if !node.is_terminal() { + // this node is not a head or a tail, so we can just cut it out of the list. update + // and put the prev and next of this node, we do `node.put` inside `insert_note`. + node.excise(); + } else if let Some(mut bag) = Bag::::get(node.bag_upper) { + // this is a head or tail, so the bag must be updated. + bag.remove_node_unchecked(&node); + bag.put(); + } else { + frame_support::defensive!( + "Node did not have a bag; BagsList is in an inconsistent state" + ); + } + + // put the node into the appropriate new bag. + let new_bag_upper = notional_bag_for::(new_score); + let mut bag = Bag::::get_or_make(new_bag_upper); + // prev, next, and bag_upper of the node are updated inside `insert_node`, also + // `node.put` is in there. + bag.insert_node_unchecked(node); + bag.put(); - /// Returns the nodes of all non-empty bags. For testing and benchmarks. - #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] - #[allow(dead_code)] - pub(crate) fn get_bags() -> Vec<(T::Score, Vec)> { - use frame_support::traits::Get as _; + Some((old_bag_upper, new_bag_upper)) + } else { + // just write the new score. + node.put(); + None + } + } - let thresholds = T::BagThresholds::get(); - let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == - Some(&T::Score::max_value()) - { - // in the event that they included it, we can just pass the iterator through unchanged. - Box::new(iter) - } else { - // otherwise, insert it here. - Box::new(iter.chain(sp_std::iter::once(T::Score::max_value()))) - }; - - iter.filter_map(|t| { - Bag::::get(t) - .map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) - }) - .collect::>() - } -} - -/// A Bag is a doubly-linked list of ids, where each id is mapped to a [`Node`]. -/// -/// Note that we maintain both head and tail pointers. While it would be possible to get away with -/// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more -/// desirable to ensure that there is some element of first-come, first-serve to the list's -/// iteration so that there's no incentive to churn ids positioning to improve the chances of -/// appearing within the ids set. -#[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen, TypeInfo)] -#[codec(mel_bound())] -#[scale_info(skip_type_params(T, I))] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] -pub struct Bag, I: 'static = ()> { - head: Option, - tail: Option, - - #[codec(skip)] - pub(super) bag_upper: T::Score, - #[codec(skip)] - _phantom: PhantomData, -} - -impl, I: 'static> Bag { - #[cfg(test)] - pub(crate) fn new( - head: Option, - tail: Option, - bag_upper: T::Score, - ) -> Self { - Self { head, tail, bag_upper, _phantom: PhantomData } - } + /// Put `heavier_id` to the position directly in front of `lighter_id`. Both ids must be in the + /// same bag and the `score_of` `lighter_id` must be less than that of `heavier_id`. + pub(crate) fn put_in_front_of( + lighter_id: &T::AccountId, + heavier_id: &T::AccountId, + ) -> Result<(), ListError> { + let lighter_node = Node::::get(&lighter_id).ok_or(ListError::NodeNotFound)?; + let heavier_node = Node::::get(&heavier_id).ok_or(ListError::NodeNotFound)?; + + ensure!(lighter_node.bag_upper == heavier_node.bag_upper, ListError::NotInSameBag); + + // this is the most expensive check, so we do it last. + ensure!( + T::ScoreProvider::score(&heavier_id) > T::ScoreProvider::score(&lighter_id), + ListError::NotHeavier + ); - /// Get a bag by its upper score. - pub(crate) fn get(bag_upper: T::Score) -> Option> { - crate::list_bags_try_get::(bag_upper) - } + // remove the heavier node from this list. Note that this removes the node from storage and + // decrements the node counter. + let _ = + Self::remove(&heavier_id).defensive_proof("both nodes have been checked to exist; qed"); - /// Get a bag by its upper score or make it, appropriately initialized. Does not check if - /// if `bag_upper` is a valid threshold. - fn get_or_make(bag_upper: T::Score) -> Bag { - Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) - } + // re-fetch `lighter_node` from storage since it may have been updated when `heavier_node` + // was removed. + let lighter_node = + Node::::get(lighter_id).defensive_ok_or_else(|| ListError::NodeNotFound)?; - /// `True` if self is empty. - fn is_empty(&self) -> bool { - self.head.is_none() && self.tail.is_none() - } + // insert `heavier_node` directly in front of `lighter_node`. This will update both nodes + // in storage and update the node counter. + Self::insert_at_unchecked(lighter_node, heavier_node); - /// Put the bag back into storage. - fn put(self) { - if self.is_empty() { - crate::list_bags_remove::(self.bag_upper); - } else { - crate::list_bags_insert::(self.bag_upper, self); + Ok(()) } - } - /// Get the head node in this bag. - fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(id)) - } + /// Insert `node` directly in front of `at`. + /// + /// WARNINGS: + /// - this is a naive function in that it does not check if `node` belongs to the same bag as + /// `at`. It is expected that the call site will check preconditions. + /// - this will panic if `at.bag_upper` is not a bag that already exists in storage. + pub fn insert_at_unchecked(mut at: Node, mut node: Node) { + // connect `node` to its new `prev`. + node.prev = at.prev.clone(); + if let Some(mut prev) = at.prev() { + prev.next = Some(node.id().clone()); + prev.put() + } - /// Get the tail node in this bag. - fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(id)) - } + // connect `node` and `at`. + node.next = Some(at.id().clone()); + at.prev = Some(node.id().clone()); - /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) - } + if node.is_terminal() { + // `node` is the new head, so we make sure the bag is updated. Note, + // since `node` is always in front of `at` we know that 1) there is always at least 2 + // nodes in the bag, and 2) only `node` could be the head and only `at` could be the + // tail. + let mut bag = Bag::::get(at.bag_upper) + .expect("given nodes must always have a valid bag. qed."); - /// Insert a new id into this bag. - /// - /// This is private on purpose because it's naive: it doesn't check whether this is the - /// appropriate bag for this id at all. Generally, use [`List::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the nodes. You still need to call - /// `self.put()` after use. - fn insert_unchecked(&mut self, id: T::AccountId, score: T::Score) { - // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long - // as this bag is the correct one, we're good. All calls to this must come after getting the - // correct [`notional_bag_for`]. - self.insert_node_unchecked(Node:: { - id, - prev: None, - next: None, - bag_upper: Zero::zero(), - score, - _phantom: PhantomData, - }); - } + if node.prev == None { + bag.head = Some(node.id().clone()) + } - /// Insert a node into this bag. - /// - /// This is private on purpose because it's naive; it doesn't check whether this is the - /// appropriate bag for this node at all. Generally, use [`List::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the node. You still need to call - /// `self.put()` after use. - fn insert_node_unchecked(&mut self, mut node: Node) { - if let Some(tail) = &self.tail { - if *tail == node.id { - // this should never happen, but this check prevents one path to a worst case - // infinite loop. - defensive!("system logic error: inserting a node who has the id of tail"); - return + bag.put() }; + + // write the updated nodes to storage. + at.put(); + node.put(); } - // re-set the `bag_upper`. Regardless of whatever the node had previously, now it is going - // to be `self.bag_upper`. - node.bag_upper = self.bag_upper; + /// Check the internal state of the list. + /// + /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) + /// is being used, after all other staking data (such as counter) has been updated. It checks: + /// + /// * there are no duplicate ids, + /// * length of this list is in sync with `ListNodes::count()`, + /// * and sanity-checks all bags and nodes. This will cascade down all the checks and makes sure + /// all bags and nodes are checked per *any* update to `List`. + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + pub(crate) fn do_try_state() -> Result<(), &'static str> { + let mut seen_in_list = BTreeSet::new(); + ensure!( + Self::iter().map(|node| node.id).all(|id| seen_in_list.insert(id)), + "duplicate identified", + ); - let id = node.id.clone(); - // update this node now, treating it as the new tail. - node.prev = self.tail.clone(); - node.next = None; - node.put(); + let iter_count = Self::iter().count() as u32; + let stored_count = crate::ListNodes::::count(); + let nodes_count = crate::ListNodes::::iter().count() as u32; + ensure!(iter_count == stored_count, "iter_count != stored_count"); + ensure!(stored_count == nodes_count, "stored_count != nodes_count"); + + crate::log!(trace, "count of nodes: {}", stored_count); + + let active_bags = { + let thresholds = T::BagThresholds::get().iter().copied(); + let thresholds: Vec = + if thresholds.clone().last() == Some(T::Score::max_value()) { + // in the event that they included it, we don't need to make any changes + thresholds.collect() + } else { + // otherwise, insert it here. + thresholds.chain(iter::once(T::Score::max_value())).collect() + }; + thresholds.into_iter().filter_map(|t| Bag::::get(t)) + }; - // update the previous tail. - if let Some(mut old_tail) = self.tail() { - old_tail.next = Some(id.clone()); - old_tail.put(); - } - self.tail = Some(id.clone()); + let _ = active_bags.clone().try_for_each(|b| b.do_try_state())?; - // ensure head exist. This is only set when the length of the bag is just 1, i.e. if this is - // the first insertion into the bag. In this case, both head and tail should point to the - // same node. - if self.head.is_none() { - self.head = Some(id); - debug_assert!(self.iter().count() == 1); - } - } + let nodes_in_bags_count = + active_bags.clone().fold(0u32, |acc, cur| acc + cur.iter().count() as u32); + ensure!(nodes_count == nodes_in_bags_count, "stored_count != nodes_in_bags_count"); - /// Remove a node from this bag. - /// - /// This is private on purpose because it doesn't check whether this bag contains the node in - /// the first place. Generally, use [`List::remove`] instead, similar to `insert_unchecked`. - /// - /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. - fn remove_node_unchecked(&mut self, node: &Node) { - // reassign neighboring nodes. - node.excise(); + crate::log!(trace, "count of active bags {}", active_bags.count()); - // clear the bag head/tail pointers as necessary. - if self.tail.as_ref() == Some(&node.id) { - self.tail = node.prev.clone(); - } - if self.head.as_ref() == Some(&node.id) { - self.head = node.next.clone(); + // check that all nodes are sane. We check the `ListNodes` storage item directly in case we + // have some "stale" nodes that are not in a bag. + for (_id, node) in crate::ListNodes::::iter() { + node.do_try_state()? + } + + Ok(()) } - } - /// Check the internal state of the bag. - /// - /// Should be called by the call-site, after any mutating operation on a bag. The call site of - /// this struct is always `List`. - /// - /// * Ensures head has no prev. - /// * Ensures tail has no next. - /// * Ensures there are no loops, traversal from head to tail is correct. - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - fn do_try_state(&self) -> Result<(), &'static str> { - frame_support::ensure!( - self.head() - .map(|head| head.prev().is_none()) - // if there is no head, then there must not be a tail, meaning that the bag is - // empty. - .unwrap_or_else(|| self.tail.is_none()), - "head has a prev" - ); - - frame_support::ensure!( - self.tail() - .map(|tail| tail.next().is_none()) - // if there is no tail, then there must not be a head, meaning that the bag is - // empty. - .unwrap_or_else(|| self.head.is_none()), - "tail has a next" - ); - - let mut seen_in_bag = BTreeSet::new(); - frame_support::ensure!( - self.iter() - .map(|node| node.id) - // each voter is only seen once, thus there is no cycle within a bag - .all(|voter| seen_in_bag.insert(voter)), - "duplicate found in bag" - ); - - Ok(()) - } + /// Returns the nodes of all non-empty bags. For testing and benchmarks. + #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] + #[allow(dead_code)] + pub(crate) fn get_bags() -> Vec<(T::Score, Vec)> { + use frame_support::traits::Get as _; + + let thresholds = T::BagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == + Some(&T::Score::max_value()) + { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter) + } else { + // otherwise, insert it here. + Box::new(iter.chain(sp_std::iter::once(T::Score::max_value()))) + }; - /// Iterate over the nodes in this bag (public for tests). - #[cfg(feature = "std")] - #[allow(dead_code)] - pub fn std_iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) + iter.filter_map(|t| { + Bag::::get(t) + .map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) + }) + .collect::>() + } } - /// Check if the bag contains a node with `id`. - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - fn contains(&self, id: &T::AccountId) -> bool { - self.iter().any(|n| n.id() == id) - } -} - -/// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. -#[derive(Encode, Decode, MaxEncodedLen, TypeInfo)] -#[codec(mel_bound())] -#[scale_info(skip_type_params(T, I))] -#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] -pub struct Node, I: 'static = ()> { - pub(crate) id: T::AccountId, - pub(crate) prev: Option, - pub(crate) next: Option, - pub(crate) bag_upper: T::Score, - pub(crate) score: T::Score, - #[codec(skip)] - pub(crate) _phantom: PhantomData, -} - -impl, I: 'static> Node { - /// Get a node by id. - pub fn get(id: &T::AccountId) -> Option> { - crate::ListNodes::::try_get(id).ok() - } + /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. + #[derive(Encode, Decode, MaxEncodedLen, TypeInfo)] + #[codec(mel_bound())] + #[scale_info(skip_type_params(T, I))] + #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] + pub struct Node, I: 'static = ()> { + pub(crate) id: T::AccountId, + pub(crate) prev: Option, + pub(crate) next: Option, + pub(crate) bag_upper: T::Score, + pub(crate) score: T::Score, + #[codec(skip)] + pub(crate) _phantom: PhantomData, + } + + impl, I: 'static> Node { + /// Get a node by id. + pub fn get(id: &T::AccountId) -> Option> { + crate::ListNodes::::try_get(id).ok() + } - /// Put the node back into storage. - fn put(self) { - crate::ListNodes::::insert(self.id.clone(), self); - } + /// Put the node back into storage. + fn put(self) { + crate::ListNodes::::insert(self.id.clone(), self); + } - /// Update neighboring nodes to point to reach other. - /// - /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call - /// `self.put`. - fn excise(&self) { - // Update previous node. - if let Some(mut prev) = self.prev() { - prev.next = self.next.clone(); - prev.put(); - } - // Update next self. - if let Some(mut next) = self.next() { - next.prev = self.prev.clone(); - next.put(); + /// Update neighboring nodes to point to reach other. + /// + /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call + /// `self.put`. + fn excise(&self) { + // Update previous node. + if let Some(mut prev) = self.prev() { + prev.next = self.next.clone(); + prev.put(); + } + // Update next self. + if let Some(mut next) = self.next() { + next.prev = self.prev.clone(); + next.put(); + } } - } - /// This is a naive function that removes a node from the `ListNodes` storage item. - /// - /// It is naive because it does not check if the node has first been removed from its bag. - fn remove_from_storage_unchecked(&self) { - crate::ListNodes::::remove(&self.id) - } + /// This is a naive function that removes a node from the `ListNodes` storage item. + /// + /// It is naive because it does not check if the node has first been removed from its bag. + fn remove_from_storage_unchecked(&self) { + crate::ListNodes::::remove(&self.id) + } - /// Get the previous node in the bag. - fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| Node::get(id)) - } + /// Get the previous node in the bag. + fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| Node::get(id)) + } - /// Get the next node in the bag. - fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| Node::get(id)) - } + /// Get the next node in the bag. + fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| Node::get(id)) + } - /// `true` when this voter is in the wrong bag. - pub fn is_misplaced(&self, current_score: T::Score) -> bool { - notional_bag_for::(current_score) != self.bag_upper - } + /// `true` when this voter is in the wrong bag. + pub fn is_misplaced(&self, current_score: T::Score) -> bool { + notional_bag_for::(current_score) != self.bag_upper + } - /// `true` when this voter is a bag head or tail. - fn is_terminal(&self) -> bool { - self.prev.is_none() || self.next.is_none() - } + /// `true` when this voter is a bag head or tail. + fn is_terminal(&self) -> bool { + self.prev.is_none() || self.next.is_none() + } - /// Get the underlying voter. - pub(crate) fn id(&self) -> &T::AccountId { - &self.id - } + /// Get the underlying voter. + pub(crate) fn id(&self) -> &T::AccountId { + &self.id + } - /// Get the current vote weight of the node. - pub(crate) fn score(&self) -> T::Score { - self.score - } + /// Get the current vote weight of the node. + pub(crate) fn score(&self) -> T::Score { + self.score + } - /// Get the underlying voter (public fo tests). - #[cfg(feature = "std")] - #[allow(dead_code)] - pub fn std_id(&self) -> &T::AccountId { - &self.id - } + /// Get the underlying voter (public fo tests). + #[cfg(feature = "std")] + #[allow(dead_code)] + pub fn std_id(&self) -> &T::AccountId { + &self.id + } - #[cfg(any(feature = "runtime-benchmarks", feature = "fuzz", test))] - pub fn set_score(&mut self, s: T::Score) { - self.score = s - } + #[cfg(any(feature = "runtime-benchmarks", feature = "fuzz", test))] + pub fn set_score(&mut self, s: T::Score) { + self.score = s + } - /// The bag this nodes belongs to (public for benchmarks). - #[cfg(feature = "runtime-benchmarks")] - #[allow(dead_code)] - pub fn bag_upper(&self) -> T::Score { - self.bag_upper - } + /// The bag this nodes belongs to (public for benchmarks). + #[cfg(feature = "runtime-benchmarks")] + #[allow(dead_code)] + pub fn bag_upper(&self) -> T::Score { + self.bag_upper + } - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - fn do_try_state(&self) -> Result<(), &'static str> { - let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + pub fn do_try_state(&self) -> Result<(), &'static str> { + let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; - let id = self.id(); + let id = self.id(); - frame_support::ensure!( - expected_bag.contains(id), - "node does not exist in the expected bag" - ); + frame_support::ensure!( + expected_bag.contains(id), + "node does not exist in the expected bag" + ); - let non_terminal_check = !self.is_terminal() && - expected_bag.head.as_ref() != Some(id) && - expected_bag.tail.as_ref() != Some(id); - let terminal_check = - expected_bag.head.as_ref() == Some(id) || expected_bag.tail.as_ref() == Some(id); - frame_support::ensure!( - non_terminal_check || terminal_check, - "a terminal node is neither its bag head or tail" - ); + let non_terminal_check = !self.is_terminal() && + expected_bag.head.as_ref() != Some(id) && + expected_bag.tail.as_ref() != Some(id); + let terminal_check = + expected_bag.head.as_ref() == Some(id) || expected_bag.tail.as_ref() == Some(id); + frame_support::ensure!( + non_terminal_check || terminal_check, + "a terminal node is neither its bag head or tail" + ); - Ok(()) + Ok(()) + } } -} -#[export_section] -mod bags_list { + /// A single node, within some bag. /// /// Nodes store links forward and back within their respective bags. #[pallet::storage] pub(crate) type ListNodes, I: 'static = ()> = - CountedStorageMap<_, Twox64Concat, T::AccountId, list::Node>; + CountedStorageMap<_, Twox64Concat, T::AccountId, Node>; /// A bag stored in storage. /// /// Stores a `Bag` struct, which stores head and tail pointers to itself. #[pallet::storage] type ListBags, I: 'static = ()> = - StorageMap<_, Twox64Concat, T::Score, list::Bag>; + StorageMap<_, Twox64Concat, T::Score, Bag>; - pub fn list_bags_get, I: 'static>(score: T::Score) -> Option> { + pub fn list_bags_get, I: 'static>(score: T::Score) -> Option> { ListBags::::get(score) } @@ -940,25 +733,216 @@ mod bags_list { ListBags::::contains_key(score) } - pub(super) fn list_bags_remove_all, I: 'static>() { - #[allow(deprecated)] - ListBags::::remove_all(None); - } + /// A Bag is a doubly-linked list of ids, where each id is mapped to a [`Node`]. + /// + /// Note that we maintain both head and tail pointers. While it would be possible to get away with + /// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more + /// desirable to ensure that there is some element of first-come, first-serve to the list's + /// iteration so that there's no incentive to churn ids positioning to improve the chances of + /// appearing within the ids set. + #[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen, TypeInfo)] + #[codec(mel_bound())] + #[scale_info(skip_type_params(T, I))] + #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] + pub struct Bag, I: 'static = ()> { + pub head: Option, + pub tail: Option, + + #[codec(skip)] + pub bag_upper: T::Score, + #[codec(skip)] + pub _phantom: PhantomData, + } + + impl, I: 'static> Bag { + #[cfg(test)] + pub(crate) fn new( + head: Option, + tail: Option, + bag_upper: T::Score, + ) -> Self { + Self { head, tail, bag_upper, _phantom: PhantomData } + } - pub(super) fn list_bags_remove, I: 'static>(score: T::Score) { - ListBags::::remove(score); - } + /// Get a bag by its upper score. + pub(crate) fn get(bag_upper: T::Score) -> Option> { + ListBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; + bag + }) + } - /// Get a bag by its upper score. - pub(super) fn list_bags_try_get, I: 'static>(bag_upper: T::Score) -> Option> { - ListBags::::try_get(bag_upper).ok().map(|mut bag| { - bag.bag_upper = bag_upper; - bag - }) - } + /// Get a bag by its upper score or make it, appropriately initialized. Does not check if + /// if `bag_upper` is a valid threshold. + pub fn get_or_make(bag_upper: T::Score) -> Bag { + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) + } - pub(super) fn list_bags_insert, I: 'static>(bag_upper: T::Score, bag: Bag) { - ListBags::::insert(bag_upper, bag); + /// `True` if self is empty. + fn is_empty(&self) -> bool { + self.head.is_none() && self.tail.is_none() + } + + /// Put the bag back into storage. + pub fn put(self) { + if self.is_empty() { + ListBags::::remove(self.bag_upper); + } else { + ListBags::::insert(self.bag_upper, self); + } + } + + /// Get the head node in this bag. + fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(id)) + } + + /// Get the tail node in this bag. + fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(id)) + } + + /// Iterate over the nodes in this bag. + pub(crate) fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Insert a new id into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this id at all. Generally, use [`List::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + pub fn insert_unchecked(&mut self, id: T::AccountId, score: T::Score) { + // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long + // as this bag is the correct one, we're good. All calls to this must come after getting the + // correct [`notional_bag_for`]. + self.insert_node_unchecked(Node:: { + id, + prev: None, + next: None, + bag_upper: Zero::zero(), + score, + _phantom: PhantomData, + }); + } + + /// Insert a node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this node at all. Generally, use [`List::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + pub fn insert_node_unchecked(&mut self, mut node: Node) { + if let Some(tail) = &self.tail { + if *tail == node.id { + // this should never happen, but this check prevents one path to a worst case + // infinite loop. + defensive!("system logic error: inserting a node who has the id of tail"); + return + }; + } + + // re-set the `bag_upper`. Regardless of whatever the node had previously, now it is going + // to be `self.bag_upper`. + node.bag_upper = self.bag_upper; + + let id = node.id.clone(); + // update this node now, treating it as the new tail. + node.prev = self.tail.clone(); + node.next = None; + node.put(); + + // update the previous tail. + if let Some(mut old_tail) = self.tail() { + old_tail.next = Some(id.clone()); + old_tail.put(); + } + self.tail = Some(id.clone()); + + // ensure head exist. This is only set when the length of the bag is just 1, i.e. if this is + // the first insertion into the bag. In this case, both head and tail should point to the + // same node. + if self.head.is_none() { + self.head = Some(id); + debug_assert!(self.iter().count() == 1); + } + } + + /// Remove a node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the node in + /// the first place. Generally, use [`List::remove`] instead, similar to `insert_unchecked`. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. + pub fn remove_node_unchecked(&mut self, node: &Node) { + // reassign neighboring nodes. + node.excise(); + + // clear the bag head/tail pointers as necessary. + if self.tail.as_ref() == Some(&node.id) { + self.tail = node.prev.clone(); + } + if self.head.as_ref() == Some(&node.id) { + self.head = node.next.clone(); + } + } + + /// Check the internal state of the bag. + /// + /// Should be called by the call-site, after any mutating operation on a bag. The call site of + /// this struct is always `List`. + /// + /// * Ensures head has no prev. + /// * Ensures tail has no next. + /// * Ensures there are no loops, traversal from head to tail is correct. + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + pub fn do_try_state(&self) -> Result<(), &'static str> { + frame_support::ensure!( + self.head() + .map(|head| head.prev().is_none()) + // if there is no head, then there must not be a tail, meaning that the bag is + // empty. + .unwrap_or_else(|| self.tail.is_none()), + "head has a prev" + ); + + frame_support::ensure!( + self.tail() + .map(|tail| tail.next().is_none()) + // if there is no tail, then there must not be a head, meaning that the bag is + // empty. + .unwrap_or_else(|| self.head.is_none()), + "tail has a next" + ); + + let mut seen_in_bag = BTreeSet::new(); + frame_support::ensure!( + self.iter() + .map(|node| node.id) + // each voter is only seen once, thus there is no cycle within a bag + .all(|voter| seen_in_bag.insert(voter)), + "duplicate found in bag" + ); + + Ok(()) + } + + /// Iterate over the nodes in this bag (public for tests). + #[cfg(feature = "std")] + #[allow(dead_code)] + pub fn std_iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Check if the bag contains a node with `id`. + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + fn contains(&self, id: &T::AccountId) -> bool { + self.iter().any(|n| n.id() == id) + } } } \ No newline at end of file diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 6629445c312ed..54ac7d84912bb 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -17,12 +17,16 @@ use super::*; use crate::{ + Config, mock::{test_utils::*, *}, - ListNodes, + List, Bag, Node, ListError, + ListNodes, notional_bag_for, list_bags_get, list_bags_contains_key }; -use frame_election_provider_support::{SortedListProvider, VoteWeight}; +use core::marker::PhantomData; +use frame_election_provider_support::{SortedListProvider, ScoreProvider, VoteWeight}; use frame_support::{assert_ok, assert_storage_noop}; +use std::iter; fn node( id: AccountId, diff --git a/frame/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs index a48b66b200e8e..1f94b6525a92f 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/bags-list/src/mock.rs @@ -161,7 +161,7 @@ impl ExtBuilder { #[cfg(test)] pub(crate) mod test_utils { use super::*; - use list::Bag; + use crate::Bag; /// Returns the ordered ids within the given bag. pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 74f1491835a32..4e049c95aaecb 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -19,7 +19,7 @@ use frame_support::{assert_noop, assert_ok, assert_storage_noop, traits::Integri use super::*; use frame_election_provider_support::{SortedListProvider, VoteWeight}; -use list::Bag; +use crate::{Bag, Node}; use mock::{test_utils::*, *}; mod pallet { @@ -153,7 +153,7 @@ mod pallet { #[test] fn wrong_rebag_errs() { ExtBuilder::default().build_and_execute(|| { - let node_3 = list::Node::::get(&3).unwrap(); + let node_3 = Node::::get(&3).unwrap(); // when account 3 is _not_ misplaced with score 500 NextVoteWeight::set(500); assert!(!node_3.is_misplaced(500)); From 9ced3181acbca5840a7837230a34db592712ba98 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 24 Apr 2023 11:07:24 +0530 Subject: [PATCH 11/55] Comments access to internal functions for now --- frame/bags-list/src/lib.rs | 4 +- frame/bags-list/src/list/mod.rs | 31 +++++++------ frame/bags-list/src/list/tests.rs | 73 +++++++++++++++---------------- frame/bags-list/src/migrations.rs | 4 +- frame/bags-list/src/tests.rs | 12 ++--- 5 files changed, 63 insertions(+), 61 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 2d0fc5b7ea01b..fa37bb351819b 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -56,7 +56,7 @@ use codec::FullCodec; use frame_election_provider_support::{ScoreProvider, SortedListProvider}; use frame_system::ensure_signed; -use sp_runtime::traits::{AtLeast32BitUnsigned, Bounded, StaticLookup}; +use sp_runtime::traits::{AtLeast32BitUnsigned, StaticLookup}; use sp_std::prelude::*; #[cfg(any(feature = "runtime-benchmarks", test))] @@ -305,7 +305,7 @@ impl, I: 'static> SortedListProvider for Pallet } fn count() -> u32 { - ListNodes::::count() + List::::count() } fn contains(id: &T::AccountId) -> bool { diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 44599a125b0a9..7f57041cd984e 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -45,7 +45,6 @@ mod bags_list { collections::{btree_map::BTreeMap, btree_set::BTreeSet}, iter, marker::PhantomData, - prelude::*, }; #[derive(Debug, PartialEq, Eq, Encode, Decode, MaxEncodedLen, TypeInfo, PalletError)] @@ -100,7 +99,7 @@ mod bags_list { #[allow(deprecated)] ListBags::::remove_all(None); #[allow(deprecated)] - crate::ListNodes::::remove_all(); + ListNodes::::remove_all(); } /// Regenerate all of the data from the given ids. @@ -156,7 +155,7 @@ mod bags_list { "not all `bag_upper` currently in storage are members of `old_thresholds`", ); debug_assert!( - crate::ListNodes::::iter() + ListNodes::::iter() .all(|(_, node)| old_thresholds.contains(&node.bag_upper)), "not all `node.bag_upper` currently in storage are members of `old_thresholds`", ); @@ -217,7 +216,7 @@ mod bags_list { // lookups. for removed_bag in removed_bags { debug_assert!( - !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), + !ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), "no id should be present in a removed bag", ); ListBags::::remove(removed_bag); @@ -228,7 +227,7 @@ mod bags_list { /// Returns `true` if the list contains `id`, otherwise returns `false`. pub(crate) fn contains(id: &T::AccountId) -> bool { - crate::ListNodes::::contains_key(id) + ListNodes::::contains_key(id) } /// Get the score of the given node, @@ -330,7 +329,7 @@ mod bags_list { id, score, bag_score, - crate::ListNodes::::count(), + ListNodes::::count(), ); Ok(()) @@ -523,8 +522,8 @@ mod bags_list { ); let iter_count = Self::iter().count() as u32; - let stored_count = crate::ListNodes::::count(); - let nodes_count = crate::ListNodes::::iter().count() as u32; + let stored_count = ListNodes::::count(); + let nodes_count = ListNodes::::iter().count() as u32; ensure!(iter_count == stored_count, "iter_count != stored_count"); ensure!(stored_count == nodes_count, "stored_count != nodes_count"); @@ -553,7 +552,7 @@ mod bags_list { // check that all nodes are sane. We check the `ListNodes` storage item directly in case we // have some "stale" nodes that are not in a bag. - for (_id, node) in crate::ListNodes::::iter() { + for (_id, node) in ListNodes::::iter() { node.do_try_state()? } @@ -584,6 +583,10 @@ mod bags_list { }) .collect::>() } + + pub fn count() -> u32 { + ListNodes::::count() + } } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. @@ -604,12 +607,12 @@ mod bags_list { impl, I: 'static> Node { /// Get a node by id. pub fn get(id: &T::AccountId) -> Option> { - crate::ListNodes::::try_get(id).ok() + ListNodes::::try_get(id).ok() } /// Put the node back into storage. - fn put(self) { - crate::ListNodes::::insert(self.id.clone(), self); + pub fn put(self) { + ListNodes::::insert(self.id.clone(), self); } /// Update neighboring nodes to point to reach other. @@ -633,7 +636,7 @@ mod bags_list { /// /// It is naive because it does not check if the node has first been removed from its bag. fn remove_from_storage_unchecked(&self) { - crate::ListNodes::::remove(&self.id) + ListNodes::::remove(&self.id) } /// Get the previous node in the bag. @@ -715,7 +718,7 @@ mod bags_list { /// /// Nodes store links forward and back within their respective bags. #[pallet::storage] - pub(crate) type ListNodes, I: 'static = ()> = + type ListNodes, I: 'static = ()> = CountedStorageMap<_, Twox64Concat, T::AccountId, Node>; /// A bag stored in storage. diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 54ac7d84912bb..7e5772184f5ff 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -15,12 +15,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::*; use crate::{ Config, mock::{test_utils::*, *}, List, Bag, Node, ListError, - ListNodes, notional_bag_for, + notional_bag_for, list_bags_get, list_bags_contains_key }; use core::marker::PhantomData; @@ -40,8 +39,8 @@ fn node( #[test] fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { - assert_eq!(ListNodes::::count(), 4); - assert_eq!(ListNodes::::iter().count(), 4); + // assert_eq!(ListNodes::::count(), 4); + // assert_eq!(ListNodes::::iter().count(), 4); // assert_eq!(ListBags::::iter().count(), 2); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); @@ -56,13 +55,13 @@ fn basic_setup_works() { Bag:: { head: Some(2), tail: Some(4), bag_upper: 0, _phantom: PhantomData } ); - assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1_000)); - assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1_000)); - assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1_000)); - assert_eq!(ListNodes::::get(1).unwrap(), node(1, None, None, 10)); + // assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1_000)); + // assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1_000)); + // assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1_000)); + // assert_eq!(ListNodes::::get(1).unwrap(), node(1, None, None, 10)); - // non-existent id does not have a storage footprint - assert_eq!(ListNodes::::get(42), None); + // // non-existent id does not have a storage footprint + // assert_eq!(ListNodes::::get(42), None); // iteration of the bags would yield: assert_eq!( @@ -260,14 +259,14 @@ mod list { #[test] fn remove_works() { let ensure_left = |id, counter| { - assert!(!ListNodes::::contains_key(id)); - assert_eq!(ListNodes::::count(), counter); - assert_eq!(ListNodes::::iter().count() as u32, counter); + // assert!(!ListNodes::::contains_key(id)); + // assert_eq!(ListNodes::::count(), counter); + // assert_eq!(ListNodes::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { // removing a non-existent id is a noop - assert!(!ListNodes::::contains_key(42)); + // assert!(!ListNodes::::contains_key(42)); assert_noop!(List::::remove(&42), ListError::NodeNotFound); // when removing a node from a bag with multiple nodes: @@ -367,14 +366,14 @@ mod list { // ensure count is in sync with `ListNodes::count()`. ExtBuilder::default().build_and_execute_no_post_check(|| { - assert_eq!(crate::ListNodes::::count(), 4); + // assert_eq!(crate::ListNodes::::count(), 4); // we do some wacky stuff here to get access to the counter, since it is (reasonably) // not exposed as mutable in any sense. #[frame_support::storage_alias] type CounterForListNodes = StorageValue, u32, frame_support::pallet_prelude::ValueQuery>; CounterForListNodes::::mutate(|counter| *counter += 1); - assert_eq!(crate::ListNodes::::count(), 5); + // assert_eq!(crate::ListNodes::::count(), 5); assert_eq!(List::::do_try_state(), Err("iter_count != stored_count")); }); @@ -412,8 +411,8 @@ mod list { }; // given - ListNodes::::insert(10, node_10_no_bag); - ListNodes::::insert(11, node_11_no_bag); + // ListNodes::::insert(10, node_10_no_bag); + // ListNodes::::insert(11, node_11_no_bag); StakingMock::set_score_of(&10, 14); StakingMock::set_score_of(&11, 15); assert!(!list_bags_contains_key::(15)); @@ -442,9 +441,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!crate::ListNodes::::contains_key(42)); + // assert!(!crate::ListNodes::::contains_key(42)); - let node_1 = crate::ListNodes::::get(&1).unwrap(); + let node_1 = Node::::get(&1).unwrap(); // when List::::insert_at_unchecked(node_1, node_42); @@ -472,9 +471,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!crate::ListNodes::::contains_key(42)); + // assert!(!crate::ListNodes::::contains_key(42)); - let node_2 = crate::ListNodes::::get(&2).unwrap(); + let node_2 = Node::::get(&2).unwrap(); // when List::::insert_at_unchecked(node_2, node_42); @@ -502,9 +501,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!crate::ListNodes::::contains_key(42)); + // assert!(!crate::ListNodes::::contains_key(42)); - let node_3 = crate::ListNodes::::get(&3).unwrap(); + let node_3 = Node::::get(&3).unwrap(); // when List::::insert_at_unchecked(node_3, node_42); @@ -532,9 +531,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!crate::ListNodes::::contains_key(42)); + // assert!(!crate::ListNodes::::contains_key(42)); - let node_4 = crate::ListNodes::::get(&4).unwrap(); + let node_4 = Node::::get(&4).unwrap(); // when List::::insert_at_unchecked(node_4, node_42); @@ -603,17 +602,17 @@ mod bags { let mut bag_10 = Bag::::get(10).unwrap(); bag_10.insert_node_unchecked(node(42, 5)); - assert_eq!( - ListNodes::::get(&42).unwrap(), - Node { - bag_upper: 10, - score: 5, - prev: Some(1), - next: None, - id: 42, - _phantom: PhantomData - } - ); + // assert_eq!( + // ListNodes::::get(&42).unwrap(), + // Node { + // bag_upper: 10, + // score: 5, + // prev: Some(1), + // next: None, + // id: 42, + // _phantom: PhantomData + // } + // ); }); } diff --git a/frame/bags-list/src/migrations.rs b/frame/bags-list/src/migrations.rs index 5f9bb8f73ac60..e5c8cad0c58ec 100644 --- a/frame/bags-list/src/migrations.rs +++ b/frame/bags-list/src/migrations.rs @@ -103,7 +103,7 @@ impl, I: 'static> OnRuntimeUpgrade for AddScore { for (_key, node) in old::ListNodes::::iter() { let score = T::ScoreProvider::score(&node.id); - let new_node = crate::Node { + let new_node = crate::Node:: { id: node.id.clone(), prev: node.prev, next: node.next, @@ -112,7 +112,7 @@ impl, I: 'static> OnRuntimeUpgrade for AddScore { _phantom: node._phantom, }; - crate::ListNodes::::insert(node.id, new_node); + new_node.put(); } return frame_support::weights::Weight::MAX diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index 4e049c95aaecb..b63e5073bbe02 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -412,7 +412,7 @@ mod pallet { // given assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); - assert!(!ListNodes::::contains_key(5)); + // assert!(!ListNodes::::contains_key(5)); // then assert_noop!( @@ -426,7 +426,7 @@ mod pallet { // given assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); - assert!(!ListNodes::::contains_key(5)); + // assert!(!ListNodes::::contains_key(5)); // then assert_noop!( @@ -612,15 +612,15 @@ mod sorted_list_provider { #[test] fn on_remove_works() { let ensure_left = |id, counter| { - assert!(!ListNodes::::contains_key(id)); + // assert!(!ListNodes::::contains_key(id)); assert_eq!(BagsList::count(), counter); - assert_eq!(ListNodes::::count(), counter); - assert_eq!(ListNodes::::iter().count() as u32, counter); + // assert_eq!(ListNodes::::count(), counter); + // assert_eq!(ListNodes::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { // it is a noop removing a non-existent id - assert!(!ListNodes::::contains_key(42)); + // assert!(!ListNodes::::contains_key(42)); assert_noop!(BagsList::on_remove(&42), ListError::NodeNotFound); // when removing a node from a bag with multiple nodes From d3f8f2462187ee771b982bc7f097a2796276f2f6 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 1 May 2023 15:19:41 +0530 Subject: [PATCH 12/55] Updates tests --- frame/bags-list/src/list/mod.rs | 49 +++++++++--------- frame/bags-list/src/list/tests.rs | 85 +++++++++++++++---------------- frame/bags-list/src/tests.rs | 20 ++++---- 3 files changed, 78 insertions(+), 76 deletions(-) diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs index 7f57041cd984e..181a5a266a281 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/list/mod.rs @@ -587,6 +587,15 @@ mod bags_list { pub fn count() -> u32 { ListNodes::::count() } + + pub fn bag_count() -> usize { + ListBags::::iter().count() + } + + #[cfg(test)] + pub fn insert_unchecked(account_id: T::AccountId, node: Node) { + ListNodes::::insert(account_id, node); + } } /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. @@ -713,29 +722,6 @@ mod bags_list { } } - - /// A single node, within some bag. - /// - /// Nodes store links forward and back within their respective bags. - #[pallet::storage] - type ListNodes, I: 'static = ()> = - CountedStorageMap<_, Twox64Concat, T::AccountId, Node>; - - /// A bag stored in storage. - /// - /// Stores a `Bag` struct, which stores head and tail pointers to itself. - #[pallet::storage] - type ListBags, I: 'static = ()> = - StorageMap<_, Twox64Concat, T::Score, Bag>; - - pub fn list_bags_get, I: 'static>(score: T::Score) -> Option> { - ListBags::::get(score) - } - - pub fn list_bags_contains_key, I: 'static>(score: T::Score) -> bool { - ListBags::::contains_key(score) - } - /// A Bag is a doubly-linked list of ids, where each id is mapped to a [`Node`]. /// /// Note that we maintain both head and tail pointers. While it would be possible to get away with @@ -775,6 +761,10 @@ mod bags_list { }) } + pub fn get_raw(score: T::Score) -> Option> { + ListBags::::get(score) + } + /// Get a bag by its upper score or make it, appropriately initialized. Does not check if /// if `bag_upper` is a valid threshold. pub fn get_or_make(bag_upper: T::Score) -> Bag { @@ -948,4 +938,17 @@ mod bags_list { } } + /// A single node, within some bag. + /// + /// Nodes store links forward and back within their respective bags. + #[pallet::storage] + type ListNodes, I: 'static = ()> = + CountedStorageMap<_, Twox64Concat, T::AccountId, Node>; + + /// A bag stored in storage. + /// + /// Stores a `Bag` struct, which stores head and tail pointers to itself. + #[pallet::storage] + type ListBags, I: 'static = ()> = + StorageMap<_, Twox64Concat, T::Score, Bag>; } \ No newline at end of file diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs index 7e5772184f5ff..5a9596d8626b3 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/list/tests.rs @@ -20,7 +20,6 @@ use crate::{ mock::{test_utils::*, *}, List, Bag, Node, ListError, notional_bag_for, - list_bags_get, list_bags_contains_key }; use core::marker::PhantomData; use frame_election_provider_support::{SortedListProvider, ScoreProvider, VoteWeight}; @@ -39,29 +38,29 @@ fn node( #[test] fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { - // assert_eq!(ListNodes::::count(), 4); - // assert_eq!(ListNodes::::iter().count(), 4); - // assert_eq!(ListBags::::iter().count(), 2); + assert_eq!(List::::count(), 4); + assert_eq!(List::::iter().count(), 4); + assert_eq!(List::::bag_count(), 2); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // the state of the bags is as expected assert_eq!( - list_bags_get(10).unwrap(), + Bag::::get_raw(10).unwrap(), Bag:: { head: Some(1), tail: Some(1), bag_upper: 0, _phantom: PhantomData } ); assert_eq!( - list_bags_get(1_000).unwrap(), + Bag::::get_raw(1_000).unwrap(), Bag:: { head: Some(2), tail: Some(4), bag_upper: 0, _phantom: PhantomData } ); - // assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1_000)); - // assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1_000)); - // assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1_000)); - // assert_eq!(ListNodes::::get(1).unwrap(), node(1, None, None, 10)); + assert_eq!(Node::::get(&2).unwrap(), node(2, None, Some(3), 1_000)); + assert_eq!(Node::::get(&3).unwrap(), node(3, Some(2), Some(4), 1_000)); + assert_eq!(Node::::get(&4).unwrap(), node(4, Some(3), None, 1_000)); + assert_eq!(Node::::get(&1).unwrap(), node(1, None, None, 10)); - // // non-existent id does not have a storage footprint - // assert_eq!(ListNodes::::get(42), None); + // non-existent id does not have a storage footprint + assert_eq!(Node::::get(&42), None); // iteration of the bags would yield: assert_eq!( @@ -259,14 +258,14 @@ mod list { #[test] fn remove_works() { let ensure_left = |id, counter| { - // assert!(!ListNodes::::contains_key(id)); - // assert_eq!(ListNodes::::count(), counter); - // assert_eq!(ListNodes::::iter().count() as u32, counter); + assert!(!List::::contains(id)); + assert_eq!(List::::count(), counter); + assert_eq!(List::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { // removing a non-existent id is a noop - // assert!(!ListNodes::::contains_key(42)); + assert!(!List::::contains(&42)); assert_noop!(List::::remove(&42), ListError::NodeNotFound); // when removing a node from a bag with multiple nodes: @@ -275,7 +274,7 @@ mod list { // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); - ensure_left(2, 3); + ensure_left(&2, 3); // when removing a node from a bag with only one node: List::::remove(&1).unwrap(); @@ -283,21 +282,21 @@ mod list { // then assert_eq!(get_list_as_ids(), vec![3, 4]); assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); - ensure_left(1, 2); + ensure_left(&1, 2); // bag 10 is removed - // assert!(!list_bags_contains_key::::contains_key(10)); + assert_eq!(Bag::::get_raw(10), None); // remove remaining ids to make sure storage cleans up as expected List::::remove(&3).unwrap(); - ensure_left(3, 1); + ensure_left(&3, 1); assert_eq!(get_list_as_ids(), vec![4]); List::::remove(&4).unwrap(); - ensure_left(4, 0); + ensure_left(&4, 0); assert_eq!(get_list_as_ids(), Vec::::new()); // bags are deleted via removals - // assert_eq!(ListBags::::iter().count(), 0); + assert_eq!(List::::bag_count(), 0); }); } @@ -366,14 +365,14 @@ mod list { // ensure count is in sync with `ListNodes::count()`. ExtBuilder::default().build_and_execute_no_post_check(|| { - // assert_eq!(crate::ListNodes::::count(), 4); + assert_eq!(List::::count(), 4); // we do some wacky stuff here to get access to the counter, since it is (reasonably) // not exposed as mutable in any sense. #[frame_support::storage_alias] type CounterForListNodes = StorageValue, u32, frame_support::pallet_prelude::ValueQuery>; CounterForListNodes::::mutate(|counter| *counter += 1); - // assert_eq!(crate::ListNodes::::count(), 5); + assert_eq!(List::::count(), 5); assert_eq!(List::::do_try_state(), Err("iter_count != stored_count")); }); @@ -411,11 +410,11 @@ mod list { }; // given - // ListNodes::::insert(10, node_10_no_bag); - // ListNodes::::insert(11, node_11_no_bag); + List::::insert_unchecked(10, node_10_no_bag); + List::::insert_unchecked(11, node_11_no_bag); StakingMock::set_score_of(&10, 14); StakingMock::set_score_of(&11, 15); - assert!(!list_bags_contains_key::(15)); + assert_eq!(Bag::::get(15), None); assert_eq!(List::::get_bags(), vec![]); // then .. this panics @@ -441,7 +440,7 @@ mod list { score: 1_000, _phantom: PhantomData, }; - // assert!(!crate::ListNodes::::contains_key(42)); + assert!(!List::::contains(&42)); let node_1 = Node::::get(&1).unwrap(); @@ -471,7 +470,7 @@ mod list { score: 1_000, _phantom: PhantomData, }; - // assert!(!crate::ListNodes::::contains_key(42)); + assert!(!List::::contains(&42)); let node_2 = Node::::get(&2).unwrap(); @@ -501,7 +500,7 @@ mod list { score: 1_000, _phantom: PhantomData, }; - // assert!(!crate::ListNodes::::contains_key(42)); + assert!(!List::::contains(&42)); let node_3 = Node::::get(&3).unwrap(); @@ -531,7 +530,7 @@ mod list { score: 1_000, _phantom: PhantomData, }; - // assert!(!crate::ListNodes::::contains_key(42)); + assert!(!List::::contains(&42)); let node_4 = Node::::get(&4).unwrap(); @@ -574,7 +573,7 @@ mod bags { .filter(|bag_upper| !vec![10, 1_000].contains(bag_upper)) .for_each(|bag_upper| { assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); - assert!(!list_bags_contains_key::(*bag_upper)); + assert_eq!(Bag::::get(*bag_upper), None); }); // when we make a pre-existing bag empty @@ -602,17 +601,17 @@ mod bags { let mut bag_10 = Bag::::get(10).unwrap(); bag_10.insert_node_unchecked(node(42, 5)); - // assert_eq!( - // ListNodes::::get(&42).unwrap(), - // Node { - // bag_upper: 10, - // score: 5, - // prev: Some(1), - // next: None, - // id: 42, - // _phantom: PhantomData - // } - // ); + assert_eq!( + Node::::get(&42).unwrap(), + Node { + bag_upper: 10, + score: 5, + prev: Some(1), + next: None, + id: 42, + _phantom: PhantomData + } + ); }); } diff --git a/frame/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs index b63e5073bbe02..288c624879ebd 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/bags-list/src/tests.rs @@ -412,7 +412,7 @@ mod pallet { // given assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); - // assert!(!ListNodes::::contains_key(5)); + assert!(!List::::contains(&5)); // then assert_noop!( @@ -426,7 +426,7 @@ mod pallet { // given assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); - // assert!(!ListNodes::::contains_key(5)); + assert!(!List::::contains(&5)); // then assert_noop!( @@ -612,15 +612,15 @@ mod sorted_list_provider { #[test] fn on_remove_works() { let ensure_left = |id, counter| { - // assert!(!ListNodes::::contains_key(id)); + assert!(!List::::contains(id)); assert_eq!(BagsList::count(), counter); - // assert_eq!(ListNodes::::count(), counter); - // assert_eq!(ListNodes::::iter().count() as u32, counter); + assert_eq!(List::::count(), counter); + assert_eq!(List::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { // it is a noop removing a non-existent id - // assert!(!ListNodes::::contains_key(42)); + assert!(!List::::contains(&42)); assert_noop!(BagsList::on_remove(&42), ListError::NodeNotFound); // when removing a node from a bag with multiple nodes @@ -629,7 +629,7 @@ mod sorted_list_provider { // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); - ensure_left(2, 3); + ensure_left(&2, 3); // when removing a node from a bag with only one node BagsList::on_remove(&1).unwrap(); @@ -637,17 +637,17 @@ mod sorted_list_provider { // then assert_eq!(get_list_as_ids(), vec![3, 4]); assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); - ensure_left(1, 2); + ensure_left(&1, 2); // when removing all remaining ids BagsList::on_remove(&4).unwrap(); assert_eq!(get_list_as_ids(), vec![3]); - ensure_left(4, 1); + ensure_left(&4, 1); BagsList::on_remove(&3).unwrap(); // then the storage is completely cleaned up assert_eq!(get_list_as_ids(), Vec::::new()); - ensure_left(3, 0); + ensure_left(&3, 0); }); } From 29c54c1d260b354add5b263e17f41a53cf3732e5 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 1 May 2023 15:42:30 +0530 Subject: [PATCH 13/55] Uncomments code --- frame/bags-list/src/lib.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index fa37bb351819b..8de2916eaff61 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -244,21 +244,21 @@ pub mod pallet { } } - // #[pallet::hooks] - // impl, I: 'static> Hooks> for Pallet { - // fn integrity_test() { - // // ensure they are strictly increasing, this also implies that duplicates are detected. - // assert!( - // T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), - // "thresholds must strictly increase, and have no duplicates", - // ); - // } - - // #[cfg(feature = "try-runtime")] - // fn try_state(_: BlockNumberFor) -> Result<(), &'static str> { - // >::try_state() - // } - // } + #[pallet::hooks] + impl, I: 'static> Hooks> for Pallet { + fn integrity_test() { + // ensure they are strictly increasing, this also implies that duplicates are detected. + assert!( + T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "thresholds must strictly increase, and have no duplicates", + ); + } + + #[cfg(feature = "try-runtime")] + fn try_state(_: BlockNumberFor) -> Result<(), &'static str> { + >::try_state() + } + } } #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] From 8b4ca57ee94a63c473ab21fa30b55ee6e8a62ac9 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 1 May 2023 15:49:24 +0530 Subject: [PATCH 14/55] Fixes test --- frame/bags-list/src/lib.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index 8de2916eaff61..f6b50fa302905 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -248,10 +248,14 @@ pub mod pallet { impl, I: 'static> Hooks> for Pallet { fn integrity_test() { // ensure they are strictly increasing, this also implies that duplicates are detected. - assert!( - T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), - "thresholds must strictly increase, and have no duplicates", - ); + for window in T::BagThresholds::get().windows(2) { + assert!(window[1] > window[0], "thresholds must strictly increase, and have no duplicates"); + } + // FAIL-CI: Following snippet fails under `import_section` but above snippet works + // assert!( + // T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), + // "thresholds must strictly increase, and have no duplicates", + // ); } #[cfg(feature = "try-runtime")] From dd119f943c11e46a174af6c83648936137529452 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Mon, 1 May 2023 16:10:47 +0530 Subject: [PATCH 15/55] Moves bags-list to separate crate --- Cargo.lock | 13 +++++++ Cargo.toml | 1 + frame/bags-list/Cargo.toml | 2 ++ frame/bags-list/src/bags-list/Cargo.toml | 34 +++++++++++++++++++ .../src/{list/mod.rs => bags-list/src/lib.rs} | 3 -- frame/bags-list/src/lib.rs | 4 ++- .../src/{list/tests.rs => tests_bags_list.rs} | 2 ++ 7 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 frame/bags-list/src/bags-list/Cargo.toml rename frame/bags-list/src/{list/mod.rs => bags-list/src/lib.rs} (99%) rename frame/bags-list/src/{list/tests.rs => tests_bags_list.rs} (99%) diff --git a/Cargo.lock b/Cargo.lock index 12a6b6715b775..932c47c38b9c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,6 +490,18 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bags-list" +version = "1.0.0" +dependencies = [ + "frame-support", + "macro_magic", + "parity-scale-codec", + "scale-info", + "sp-runtime", + "sp-std", +] + [[package]] name = "base-x" version = "0.2.11" @@ -5790,6 +5802,7 @@ dependencies = [ name = "pallet-bags-list" version = "4.0.0-dev" dependencies = [ + "bags-list", "frame-benchmarking", "frame-election-provider-support", "frame-support", diff --git a/Cargo.toml b/Cargo.toml index 82e2264a3a2c6..3b313a0b93721 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,6 +80,7 @@ members = [ "frame/authorship", "frame/babe", "frame/bags-list", + "frame/bags-list/src/bags-list", "frame/bags-list/fuzzer", "frame/bags-list/remote-tests", "frame/balances", diff --git a/frame/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml index 1219336a7b2bb..78b49a9993c03 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/bags-list/Cargo.toml @@ -15,6 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] # parity codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } +bags-list = { version = "1.0.0", default-features = false, path = "./src/bags-list" } # primitives sp-runtime = { version = "7.0.0", default-features = false, path = "../../primitives/runtime" } @@ -60,6 +61,7 @@ std = [ "frame-system/std", "frame-election-provider-support/std", "log/std", + "bags-list/std" ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", diff --git a/frame/bags-list/src/bags-list/Cargo.toml b/frame/bags-list/src/bags-list/Cargo.toml new file mode 100644 index 0000000000000..74b58c8b5e265 --- /dev/null +++ b/frame/bags-list/src/bags-list/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "bags-list" +version = "1.0.0" +authors = ["Parity Technologies "] +edition = "2021" +license = "Apache-2.0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME bags list" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +# parity +codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } +scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } + +# primitives +sp-runtime = { version = "7.0.0", default-features = false, path = "../../../../primitives/runtime" } +sp-std = { version = "5.0.0", default-features = false, path = "../../../../primitives/std" } + +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../support" } + +macro_magic = "0.3.1" + +[features] +default = ["std"] +std = [ + "scale-info/std", + "codec/std", + "sp-runtime/std", + "sp-std/std", + "frame-support/std" +] \ No newline at end of file diff --git a/frame/bags-list/src/list/mod.rs b/frame/bags-list/src/bags-list/src/lib.rs similarity index 99% rename from frame/bags-list/src/list/mod.rs rename to frame/bags-list/src/bags-list/src/lib.rs index 181a5a266a281..277533d4e6930 100644 --- a/frame/bags-list/src/list/mod.rs +++ b/frame/bags-list/src/bags-list/src/lib.rs @@ -26,9 +26,6 @@ use frame_support::pallet_macros::*; -#[cfg(test)] -mod tests; - #[export_section] mod bags_list { use codec::{Decode, Encode, MaxEncodedLen}; diff --git a/frame/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs index f6b50fa302905..c7dcc05a9e79e 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/bags-list/src/lib.rs @@ -58,16 +58,18 @@ use frame_election_provider_support::{ScoreProvider, SortedListProvider}; use frame_system::ensure_signed; use sp_runtime::traits::{AtLeast32BitUnsigned, StaticLookup}; use sp_std::prelude::*; +use bags_list::*; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; -mod list; pub mod migrations; #[cfg(any(test, feature = "fuzz"))] pub mod mock; #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_bags_list; pub mod weights; pub use pallet::*; diff --git a/frame/bags-list/src/list/tests.rs b/frame/bags-list/src/tests_bags_list.rs similarity index 99% rename from frame/bags-list/src/list/tests.rs rename to frame/bags-list/src/tests_bags_list.rs index 5a9596d8626b3..2b16e94f2451c 100644 --- a/frame/bags-list/src/list/tests.rs +++ b/frame/bags-list/src/tests_bags_list.rs @@ -26,6 +26,8 @@ use frame_election_provider_support::{SortedListProvider, ScoreProvider, VoteWei use frame_support::{assert_ok, assert_storage_noop}; use std::iter; +//TODO: These tests should move under `bags-list` crate + fn node( id: AccountId, prev: Option, From d4b1969c43b043b8318211987bf66ac3b6f81330 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Thu, 25 May 2023 12:00:45 +0530 Subject: [PATCH 16/55] Initial setup --- Cargo.lock | 67 +++++ Cargo.toml | 1 + frame/examples/split/Cargo.toml | 46 ++++ frame/examples/split/README.md | 240 ++++++++++++++++++ frame/examples/split/src/lib.rs | 103 ++++++++ frame/examples/split/src/mock.rs | 59 +++++ frame/examples/split/src/storages.rs | 12 + frame/examples/split/src/tests.rs | 27 ++ frame/examples/split/src/weights.rs | 91 +++++++ frame/support/Cargo.toml | 1 + frame/support/procedural/Cargo.toml | 1 + frame/support/procedural/src/lib.rs | 28 ++ .../procedural/src/pallet/expand/storage.rs | 5 +- frame/support/src/lib.rs | 7 + 14 files changed, 687 insertions(+), 1 deletion(-) create mode 100644 frame/examples/split/Cargo.toml create mode 100644 frame/examples/split/README.md create mode 100644 frame/examples/split/src/lib.rs create mode 100644 frame/examples/split/src/mock.rs create mode 100644 frame/examples/split/src/storages.rs create mode 100644 frame/examples/split/src/tests.rs create mode 100644 frame/examples/split/src/weights.rs diff --git a/Cargo.lock b/Cargo.lock index 7bffd9e169159..840b290139db6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2679,6 +2679,7 @@ dependencies = [ "impl-trait-for-tuples", "k256", "log", + "macro_magic", "once_cell", "parity-scale-codec", "paste", @@ -2712,6 +2713,7 @@ dependencies = [ "derive-syn-parse", "frame-support-procedural-tools", "itertools", + "macro_magic", "proc-macro-warning", "proc-macro2", "quote", @@ -4728,6 +4730,53 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_magic" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f3703e9d296140171bbd6a6fa560e6059a35299ee3ce8c5bd646afa0c3992" +dependencies = [ + "macro_magic_core", + "macro_magic_macros", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "macro_magic_core" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e060da9399c1535b3f190084d7c6121bb3355b70c610110cff3f4978526f54e2" +dependencies = [ + "derive-syn-parse", + "macro_magic_core_macros", + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "macro_magic_core_macros" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fb9865e03b9641e57448b9743915eadd0f642e41a41c4d9eaa8b5b861d887b0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.16", +] + +[[package]] +name = "macro_magic_macros" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68cf083ba5ba151ce3230a7c687cb5c890498adae59c85f3be7e8e741a6c9f65" +dependencies = [ + "macro_magic_core", + "quote", + "syn 2.0.16", +] + [[package]] name = "maplit" version = "1.0.2" @@ -6392,6 +6441,24 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-example-split" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log", + "macro_magic", + "pallet-balances", + "parity-scale-codec", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-fast-unstake" version = "4.0.0-dev" diff --git a/Cargo.toml b/Cargo.toml index 0ac6a9dfb57e4..357cc4406671e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -109,6 +109,7 @@ members = [ "frame/examples/basic", "frame/examples/offchain-worker", "frame/examples/dev-mode", + "frame/examples/split", "frame/executive", "frame/nis", "frame/grandpa", diff --git a/frame/examples/split/Cargo.toml b/frame/examples/split/Cargo.toml new file mode 100644 index 0000000000000..6c83919eb38ac --- /dev/null +++ b/frame/examples/split/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "pallet-example-split" +version = "4.0.0-dev" +authors = ["Parity Technologies "] +edition = "2021" +license = "MIT-0" +homepage = "https://substrate.io" +repository = "https://github.com/paritytech/substrate/" +description = "FRAME example pallet" +readme = "README.md" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false } +log = { version = "0.4.17", default-features = false } +scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } +frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../benchmarking" } +frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } +frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } +pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" } +sp-io = { version = "7.0.0", default-features = false, path = "../../../primitives/io" } +sp-runtime = { version = "7.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "5.0.0", default-features = false, path = "../../../primitives/std" } +macro_magic = "0.3.1" + +[dev-dependencies] +sp-core = { version = "7.0.0", default-features = false, path = "../../../primitives/core" } + +[features] +default = ["std"] +std = [ + "codec/std", + "frame-benchmarking?/std", + "frame-support/std", + "frame-system/std", + "log/std", + "pallet-balances/std", + "scale-info/std", + "sp-io/std", + "sp-runtime/std", + "sp-std/std", +] +runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] +try-runtime = ["frame-support/try-runtime"] diff --git a/frame/examples/split/README.md b/frame/examples/split/README.md new file mode 100644 index 0000000000000..2af50573f8075 --- /dev/null +++ b/frame/examples/split/README.md @@ -0,0 +1,240 @@ + +# Basic Example Pallet + + +The Example: A simple example of a FRAME pallet demonstrating +concepts, APIs and structures common to most FRAME runtimes. + +Run `cargo doc --package pallet-example-basic --open` to view this pallet's documentation. + +**This pallet serves as an example and is not meant to be used in production.** + +### Documentation Guidelines: + + + +
    +
  • Documentation comments (i.e. /// comment) - should + accompany pallet functions and be restricted to the pallet interface, + not the internals of the pallet implementation. Only state inputs, + outputs, and a brief description that mentions whether calling it + requires root, but without repeating the source code details. + Capitalize the first word of each documentation comment and end it with + a full stop. See + Generic example of annotating source code with documentation comments
    • +
  • Self-documenting code - Try to refactor code to be self-documenting.
    • +
  • Code comments - Supplement complex code with a brief explanation, not every line of code.
    • +
  • Identifiers - surround by backticks (i.e. INHERENT_IDENTIFIER, InherentType, + u64)
    • +
  • Usage scenarios - should be simple doctests. The compiler should ensure they stay valid.
    • +
  • Extended tutorials - should be moved to external files and refer to.
    • + +
  • Mandatory - include all of the sections/subsections where MUST is specified.
    • +
  • Optional - optionally include sections/subsections where CAN is specified.
    • +
+ +### Documentation Template:
+ +Copy and paste this template from frame/examples/basic/src/lib.rs into file +`frame//src/lib.rs` of your own custom pallet and complete it. +

+// Add heading with custom pallet name
+
+\#  Pallet
+
+// Add simple description
+
+// Include the following links that shows what trait needs to be implemented to use the pallet
+// and the supported dispatchables that are documented in the Call enum.
+
+- \[`::Config`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/trait.Config.html)
+- \[`Call`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/enum.Call.html)
+- \[`Module`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/struct.Module.html)
+
+\## Overview
+
+
+// Short description of pallet's purpose.
+// Links to Traits that should be implemented.
+// What this pallet is for.
+// What functionality the pallet provides.
+// When to use the pallet (use case examples).
+// How it is used.
+// Inputs it uses and the source of each input.
+// Outputs it produces.
+
+
+
+
+\## Terminology
+
+// Add terminology used in the custom pallet. Include concepts, storage items, or actions that you think
+// deserve to be noted to give context to the rest of the documentation or pallet usage. The author needs to
+// use some judgment about what is included. We don't want a list of every storage item nor types - the user
+// can go to the code for that. For example, "transfer fee" is obvious and should not be included, but
+// "free balance" and "reserved balance" should be noted to give context to the pallet.
+// Please do not link to outside resources. The reference docs should be the ultimate source of truth.
+
+
+
+\## Goals
+
+// Add goals that the custom pallet is designed to achieve.
+
+
+
+\### Scenarios
+
+
+
+\#### 
+
+// Describe requirements prior to interacting with the custom pallet.
+// Describe the process of interacting with the custom pallet for this scenario and public API functions used.
+
+\## Interface
+
+\### Supported Origins
+
+// What origins are used and supported in this pallet (root, signed, none)
+// i.e. root when \`ensure_root\` used
+// i.e. none when \`ensure_none\` used
+// i.e. signed when \`ensure_signed\` used
+
+\`inherent\` 
+
+
+
+
+\### Types
+
+// Type aliases. Include any associated types and where the user would typically define them.
+
+\`ExampleType\` 
+
+
+
+// Reference documentation of aspects such as `storageItems` and `dispatchable` functions should only be
+// included in the https://docs.rs Rustdocs for Substrate and not repeated in the README file.
+
+\### Dispatchable Functions
+
+
+
+// A brief description of dispatchable functions and a link to the rustdoc with their actual documentation.
+
+// MUST have link to Call enum
+// MUST have origin information included in function doc
+// CAN have more info up to the user
+
+\### Public Functions
+
+
+
+// A link to the rustdoc and any notes about usage in the pallet, not for specific functions.
+// For example, in the Balances Pallet: "Note that when using the publicly exposed functions,
+// you (the runtime developer) are responsible for implementing any necessary checks
+// (e.g. that the sender is the signer) before calling a function that will affect storage."
+
+
+
+// It is up to the writer of the respective pallet (with respect to how much information to provide).
+
+\#### Public Inspection functions - Immutable (getters)
+
+// Insert a subheading for each getter function signature
+
+\##### \`example_getter_name()\`
+
+// What it returns
+// Why, when, and how often to call it
+// When it could panic or error
+// When safety issues to consider
+
+\#### Public Mutable functions (changing state)
+
+// Insert a subheading for each setter function signature
+
+\##### \`example_setter_name(origin, parameter_name: T::ExampleType)\`
+
+// What state it changes
+// Why, when, and how often to call it
+// When it could panic or error
+// When safety issues to consider
+// What parameter values are valid and why
+
+\### Storage Items
+
+// Explain any storage items included in this pallet
+
+\### Digest Items
+
+// Explain any digest items included in this pallet
+
+\### Inherent Data
+
+// Explain what inherent data (if any) is defined in the pallet and any other related types
+
+\### Events:
+
+// Insert events for this pallet if any
+
+\### Errors:
+
+// Explain what generates errors
+
+\## Usage
+
+// Insert 2-3 examples of usage and code snippets that show how to
+// use  Pallet in a custom pallet.
+
+\### Prerequisites
+
+// Show how to include necessary imports for  and derive
+// your pallet configuration trait with the `INSERT_CUSTOM_PALLET_NAME` trait.
+
+\```rust
+use ;
+
+pub trait Config: ::Config { }
+\```
+
+\### Simple Code Snippet
+
+// Show a simple example (e.g. how to query a public getter function of )
+
+\### Example from FRAME
+
+// Show a usage example in an actual runtime
+
+// See:
+// - Substrate TCR https://github.com/parity-samples/substrate-tcr
+// - Substrate Kitties https://shawntabrizi.github.io/substrate-collectables-workshop/#/
+
+\## Genesis Config
+
+
+
+\## Dependencies
+
+// Dependencies on other FRAME pallets and the genesis config should be mentioned,
+// but not the Rust Standard Library.
+// Genesis configuration modifications that may be made to incorporate this pallet
+// Interaction with other pallets
+
+
+
+\## Related Pallets
+
+// Interaction with other pallets in the form of a bullet point list
+
+\## References
+
+
+
+// Links to reference material, if applicable. For example, Phragmen, W3F research, etc.
+// that the implementation is based on.
+

+ +License: MIT-0 diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs new file mode 100644 index 0000000000000..d1779bf59abc3 --- /dev/null +++ b/frame/examples/split/src/lib.rs @@ -0,0 +1,103 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +/// Edit this file to define custom logic or remove it if it is not needed. +/// Learn more about FRAME and the core library of Substrate FRAME pallets: +/// +pub use pallet::*; + +#[cfg(test)] +mod mock; + +#[cfg(test)] +mod tests; + +#[cfg(feature = "runtime-benchmarks")] +mod benchmarking; +mod storages; +pub mod weights; +pub use weights::*; +use frame_support::pallet_macros::*; + +#[import_section(storages)] +#[frame_support::pallet] +pub mod pallet { + use super::*; + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + /// Configure the pallet by specifying the parameters and types on which it depends. + #[pallet::config] + pub trait Config: frame_system::Config { + /// Because this pallet emits events, it depends on the runtime's definition of an event. + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + /// Type representing the weight of this pallet + type WeightInfo: WeightInfo; + } + + // Pallets use events to inform users when important changes are made. + // https://docs.substrate.io/main-docs/build/events-errors/ + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// Event documentation should end with an array that provides descriptive names for event + /// parameters. [something, who] + SomethingStored { something: u32, who: T::AccountId }, + } + + // Errors inform users that something went wrong. + #[pallet::error] + pub enum Error { + /// Error names should be descriptive. + NoneValue, + /// Errors should have helpful documentation associated with them. + StorageOverflow, + } + + // Dispatchable functions allows users to interact with the pallet and invoke state changes. + // These functions materialize as "extrinsics", which are often compared to transactions. + // Dispatchable functions must be annotated with a weight and must return a DispatchResult. + #[pallet::call] + impl Pallet { + /// An example dispatchable that takes a singles value as a parameter, writes the value to + /// storage and emits an event. This function must be dispatched by a signed extrinsic. + #[pallet::call_index(0)] + #[pallet::weight(T::WeightInfo::do_something())] + pub fn do_something(origin: OriginFor, something: u32) -> DispatchResult { + // Check that the extrinsic was signed and get the signer. + // This function will return an error if the extrinsic is not signed. + // https://docs.substrate.io/main-docs/build/origins/ + let who = ensure_signed(origin)?; + + // Update storage. + >::put(something); + + // Emit an event. + Self::deposit_event(Event::SomethingStored { something, who }); + // Return a successful DispatchResultWithPostInfo + Ok(()) + } + + /// An example dispatchable that may throw a custom error. + #[pallet::call_index(1)] + #[pallet::weight(T::WeightInfo::cause_error())] + pub fn cause_error(origin: OriginFor) -> DispatchResult { + let _who = ensure_signed(origin)?; + + // Read a value from storage. + match >::get() { + // Return an error if the value has not been set. + None => return Err(Error::::NoneValue.into()), + Some(old) => { + // Increment the value read from storage; will error in the event of overflow. + let new = old.checked_add(1).ok_or(Error::::StorageOverflow)?; + // Update the value in storage with the incremented result. + >::put(new); + Ok(()) + }, + } + } + } +} diff --git a/frame/examples/split/src/mock.rs b/frame/examples/split/src/mock.rs new file mode 100644 index 0000000000000..b4d6905378a5d --- /dev/null +++ b/frame/examples/split/src/mock.rs @@ -0,0 +1,59 @@ +use crate as pallet_template; +use frame_support::traits::{ConstU16, ConstU64}; +use sp_core::H256; +use sp_runtime::{ + testing::Header, + traits::{BlakeTwo256, IdentityLookup}, +}; + +type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; +type Block = frame_system::mocking::MockBlock; + +// Configure a mock runtime to test the pallet. +frame_support::construct_runtime!( + pub enum Test where + Block = Block, + NodeBlock = Block, + UncheckedExtrinsic = UncheckedExtrinsic, + { + System: frame_system, + TemplateModule: pallet_template, + } +); + +impl frame_system::Config for Test { + type BaseCallFilter = frame_support::traits::Everything; + type BlockWeights = (); + type BlockLength = (); + type DbWeight = (); + type RuntimeOrigin = RuntimeOrigin; + type RuntimeCall = RuntimeCall; + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = BlakeTwo256; + type AccountId = u64; + type Lookup = IdentityLookup; + type Header = Header; + type RuntimeEvent = RuntimeEvent; + type BlockHashCount = ConstU64<250>; + type Version = (); + type PalletInfo = PalletInfo; + type AccountData = (); + type OnNewAccount = (); + type OnKilledAccount = (); + type SystemWeightInfo = (); + type SS58Prefix = ConstU16<42>; + type OnSetCode = (); + type MaxConsumers = frame_support::traits::ConstU32<16>; +} + +impl pallet_template::Config for Test { + type RuntimeEvent = RuntimeEvent; + type WeightInfo = (); +} + +// Build genesis storage according to the mock runtime. +pub fn new_test_ext() -> sp_io::TestExternalities { + frame_system::GenesisConfig::default().build_storage::().unwrap().into() +} diff --git a/frame/examples/split/src/storages.rs b/frame/examples/split/src/storages.rs new file mode 100644 index 0000000000000..bceb750a7143c --- /dev/null +++ b/frame/examples/split/src/storages.rs @@ -0,0 +1,12 @@ +use frame_support::pallet_macros::*; + +#[export_section] +mod storages { + // The pallet's runtime storage items. + // https://docs.substrate.io/main-docs/build/runtime-storage/ + #[pallet::storage] + #[pallet::getter(fn something)] + // Learn more about declaring storage items: + // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items + pub type Something = StorageValue<_, u32>; +} \ No newline at end of file diff --git a/frame/examples/split/src/tests.rs b/frame/examples/split/src/tests.rs new file mode 100644 index 0000000000000..7c2b853ee4dc5 --- /dev/null +++ b/frame/examples/split/src/tests.rs @@ -0,0 +1,27 @@ +use crate::{mock::*, Error, Event}; +use frame_support::{assert_noop, assert_ok}; + +#[test] +fn it_works_for_default_value() { + new_test_ext().execute_with(|| { + // Go past genesis block so events get deposited + System::set_block_number(1); + // Dispatch a signed extrinsic. + assert_ok!(TemplateModule::do_something(RuntimeOrigin::signed(1), 42)); + // Read pallet storage and assert an expected result. + assert_eq!(TemplateModule::something(), Some(42)); + // Assert that the correct event was deposited + System::assert_last_event(Event::SomethingStored { something: 42, who: 1 }.into()); + }); +} + +#[test] +fn correct_error_for_none_value() { + new_test_ext().execute_with(|| { + // Ensure the expected error is thrown when no value is present. + assert_noop!( + TemplateModule::cause_error(RuntimeOrigin::signed(1)), + Error::::NoneValue + ); + }); +} diff --git a/frame/examples/split/src/weights.rs b/frame/examples/split/src/weights.rs new file mode 100644 index 0000000000000..e8fbc09bad8e9 --- /dev/null +++ b/frame/examples/split/src/weights.rs @@ -0,0 +1,91 @@ + +//! Autogenerated weights for pallet_template +//! +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev +//! DATE: 2023-04-06, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! WORST CASE MAP SIZE: `1000000` +//! HOSTNAME: `Alexs-MacBook-Pro-2.local`, CPU: `` +//! EXECUTION: Some(Wasm), WASM-EXECUTION: Compiled, CHAIN: Some("dev"), DB CACHE: 1024 + +// Executed Command: +// ../../target/release/node-template +// benchmark +// pallet +// --chain +// dev +// --pallet +// pallet_template +// --extrinsic +// * +// --steps=50 +// --repeat=20 +// --execution=wasm +// --wasm-execution=compiled +// --output +// pallets/template/src/weights.rs +// --template +// ../../.maintain/frame-weight-template.hbs + +#![cfg_attr(rustfmt, rustfmt_skip)] +#![allow(unused_parens)] +#![allow(unused_imports)] + +use frame_support::{traits::Get, weights::{Weight, constants::RocksDbWeight}}; +use core::marker::PhantomData; + +/// Weight functions needed for pallet_template. +pub trait WeightInfo { + fn do_something() -> Weight; + fn cause_error() -> Weight; +} + +/// Weights for pallet_template using the Substrate node and recommended hardware. +pub struct SubstrateWeight(PhantomData); +impl WeightInfo for SubstrateWeight { + /// Storage: TemplateModule Something (r:0 w:1) + /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + fn do_something() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 8_000_000 picoseconds. + Weight::from_parts(9_000_000, 0) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } + /// Storage: TemplateModule Something (r:1 w:1) + /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + fn cause_error() -> Weight { + // Proof Size summary in bytes: + // Measured: `32` + // Estimated: `1489` + // Minimum execution time: 6_000_000 picoseconds. + Weight::from_parts(6_000_000, 1489) + .saturating_add(T::DbWeight::get().reads(1_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } +} + +// For backwards compatibility and tests +impl WeightInfo for () { + /// Storage: TemplateModule Something (r:0 w:1) + /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + fn do_something() -> Weight { + // Proof Size summary in bytes: + // Measured: `0` + // Estimated: `0` + // Minimum execution time: 8_000_000 picoseconds. + Weight::from_parts(9_000_000, 0) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } + /// Storage: TemplateModule Something (r:1 w:1) + /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + fn cause_error() -> Weight { + // Proof Size summary in bytes: + // Measured: `32` + // Estimated: `1489` + // Minimum execution time: 6_000_000 picoseconds. + Weight::from_parts(6_000_000, 1489) + .saturating_add(RocksDbWeight::get().reads(1_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } +} diff --git a/frame/support/Cargo.toml b/frame/support/Cargo.toml index 6ed67c8081b5b..7329cc68e7e45 100644 --- a/frame/support/Cargo.toml +++ b/frame/support/Cargo.toml @@ -29,6 +29,7 @@ sp-staking = { version = "4.0.0-dev", default-features = false, path = "../../pr sp-weights = { version = "4.0.0", default-features = false, path = "../../primitives/weights" } sp-debug-derive = { default-features = false, path = "../../primitives/debug-derive" } tt-call = "1.0.8" +macro_magic = "0.3.1" frame-support-procedural = { version = "4.0.0-dev", default-features = false, path = "./procedural" } paste = "1.0" once_cell = { version = "1", default-features = false, optional = true } diff --git a/frame/support/procedural/Cargo.toml b/frame/support/procedural/Cargo.toml index e6fe140b516a1..4f8d6758926d0 100644 --- a/frame/support/procedural/Cargo.toml +++ b/frame/support/procedural/Cargo.toml @@ -24,6 +24,7 @@ quote = "1.0.26" syn = { version = "2.0.16", features = ["full"] } frame-support-procedural-tools = { version = "4.0.0-dev", path = "./tools" } proc-macro-warning = { version = "0.3.0", default-features = false } +macro_magic = { version = "0.3.1", features = ["proc_support"] } [features] default = ["std"] diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 25df8410b02d1..f8a0b7dc48faf 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -38,7 +38,10 @@ mod tt_macro; use proc_macro::TokenStream; use quote::quote; +use macro_magic::*; +use syn::{parse_macro_input, ItemMod}; use std::{cell::RefCell, str::FromStr}; +use quote::ToTokens; pub(crate) use storage::INHERENT_INSTANCE_NAME; thread_local! { @@ -1421,3 +1424,28 @@ pub fn origin(_: TokenStream, _: TokenStream) -> TokenStream { pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { pallet_macro_stub() } + +#[proc_macro_attribute] +pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { + match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { + Ok(tokens) => tokens.into(), + Err(err) => err.to_compile_error().into(), + } +} + +#[import_tokens_attr] +#[proc_macro_attribute] +pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { + let foreign_mod = parse_macro_input!(attr as ItemMod); + let mut internal_mod = parse_macro_input!(tokens as ItemMod); + + if let Some(ref mut content) = internal_mod.content { + if let Some(foreign_content) = foreign_mod.content { + content.1.extend(foreign_content.1); + } + } + + quote! { + #internal_mod + }.into() +} \ No newline at end of file diff --git a/frame/support/procedural/src/pallet/expand/storage.rs b/frame/support/procedural/src/pallet/expand/storage.rs index c742ddcd25fbc..b64d47f8832b1 100644 --- a/frame/support/procedural/src/pallet/expand/storage.rs +++ b/frame/support/procedural/src/pallet/expand/storage.rs @@ -25,6 +25,7 @@ use crate::{ use quote::ToTokens; use std::{collections::HashMap, ops::IndexMut}; use syn::spanned::Spanned; +use proc_macro2::Span; /// Generate the prefix_ident related to the storage. /// prefix_ident is used for the prefix struct to be given to storage as first generic param. @@ -348,7 +349,9 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { let cfg_attrs = &storage.cfg_attrs; - quote::quote_spanned!(storage.attr_span => + // Since we are using `entries` defined later, we need to specify `call_site` to disable + // macro hygiene. + quote::quote_spanned!(Span::call_site() => #(#cfg_attrs)* { <#full_ident as #frame_support::storage::StorageEntryMetadataBuilder>::build_metadata( diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 3cd8378be45d1..0a73271108cb5 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -822,6 +822,10 @@ macro_rules! assert_error_encoded_size { #[doc(hidden)] pub use serde::{Deserialize, Serialize}; +#[doc(hidden)] +#[cfg(not(no_std))] +pub use macro_magic; + #[cfg(test)] pub mod tests { use super::*; @@ -2888,6 +2892,9 @@ pub mod pallet_macros { storage_prefix, storage_version, type_value, unbounded, validate_unsigned, weight, whitelist_storage, }; + #[macro_magic::use_attr] + pub use frame_support_procedural::import_section; + pub use frame_support_procedural::export_section; } // Generate a macro that will enable/disable code based on `std` feature being active. From 596cff526d6535d580f6eb7205fc70a7f1c9950c Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Thu, 25 May 2023 12:02:08 +0530 Subject: [PATCH 17/55] Removes bags-list changes --- frame/bags-list/src/bags-list/Cargo.toml | 34 - frame/bags-list/src/bags-list/src/lib.rs | 951 ------------------ frame/{ => balances}/bags-list/Cargo.toml | 3 - .../bags-list/fuzzer/.gitignore | 0 .../bags-list/fuzzer/Cargo.toml | 0 .../bags-list/fuzzer/src/main.rs | 0 .../bags-list/remote-tests/Cargo.toml | 0 .../bags-list/remote-tests/src/lib.rs | 0 .../bags-list/remote-tests/src/migration.rs | 0 .../bags-list/remote-tests/src/snapshot.rs | 0 .../bags-list/remote-tests/src/try_state.rs | 0 .../bags-list/src/benchmarks.rs | 2 +- frame/{ => balances}/bags-list/src/lib.rs | 61 +- frame/balances/bags-list/src/list/mod.rs | 920 +++++++++++++++++ .../bags-list/src/list/tests.rs} | 98 +- .../bags-list/src/migrations.rs | 13 +- frame/{ => balances}/bags-list/src/mock.rs | 9 +- frame/{ => balances}/bags-list/src/tests.rs | 24 +- frame/{ => balances}/bags-list/src/weights.rs | 0 19 files changed, 1034 insertions(+), 1081 deletions(-) delete mode 100644 frame/bags-list/src/bags-list/Cargo.toml delete mode 100644 frame/bags-list/src/bags-list/src/lib.rs rename frame/{ => balances}/bags-list/Cargo.toml (95%) rename frame/{ => balances}/bags-list/fuzzer/.gitignore (100%) rename frame/{ => balances}/bags-list/fuzzer/Cargo.toml (100%) rename frame/{ => balances}/bags-list/fuzzer/src/main.rs (100%) rename frame/{ => balances}/bags-list/remote-tests/Cargo.toml (100%) rename frame/{ => balances}/bags-list/remote-tests/src/lib.rs (100%) rename frame/{ => balances}/bags-list/remote-tests/src/migration.rs (100%) rename frame/{ => balances}/bags-list/remote-tests/src/snapshot.rs (100%) rename frame/{ => balances}/bags-list/remote-tests/src/try_state.rs (100%) rename frame/{ => balances}/bags-list/src/benchmarks.rs (99%) rename frame/{ => balances}/bags-list/src/lib.rs (89%) create mode 100644 frame/balances/bags-list/src/list/mod.rs rename frame/{bags-list/src/tests_bags_list.rs => balances/bags-list/src/list/tests.rs} (91%) rename frame/{ => balances}/bags-list/src/migrations.rs (92%) rename frame/{ => balances}/bags-list/src/mock.rs (95%) rename frame/{ => balances}/bags-list/src/tests.rs (97%) rename frame/{ => balances}/bags-list/src/weights.rs (100%) diff --git a/frame/bags-list/src/bags-list/Cargo.toml b/frame/bags-list/src/bags-list/Cargo.toml deleted file mode 100644 index 74b58c8b5e265..0000000000000 --- a/frame/bags-list/src/bags-list/Cargo.toml +++ /dev/null @@ -1,34 +0,0 @@ -[package] -name = "bags-list" -version = "1.0.0" -authors = ["Parity Technologies "] -edition = "2021" -license = "Apache-2.0" -homepage = "https://substrate.io" -repository = "https://github.com/paritytech/substrate/" -description = "FRAME bags list" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -# parity -codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } -scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } - -# primitives -sp-runtime = { version = "7.0.0", default-features = false, path = "../../../../primitives/runtime" } -sp-std = { version = "5.0.0", default-features = false, path = "../../../../primitives/std" } - -frame-support = { version = "4.0.0-dev", default-features = false, path = "../../../support" } - -macro_magic = "0.3.1" - -[features] -default = ["std"] -std = [ - "scale-info/std", - "codec/std", - "sp-runtime/std", - "sp-std/std", - "frame-support/std" -] \ No newline at end of file diff --git a/frame/bags-list/src/bags-list/src/lib.rs b/frame/bags-list/src/bags-list/src/lib.rs deleted file mode 100644 index 277533d4e6930..0000000000000 --- a/frame/bags-list/src/bags-list/src/lib.rs +++ /dev/null @@ -1,951 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 - -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! Implementation of a "bags list": a semi-sorted list where ordering granularity is dictated by -//! configurable thresholds that delineate the boundaries of bags. It uses a pattern of composite -//! data structures, where multiple storage items are masked by one outer API. See -//! [`crate::ListNodes`], [`crate::ListBags`] for more information. -//! -//! The outer API of this module is the [`List`] struct. It wraps all acceptable operations on top -//! of the aggregate linked list. All operations with the bags list should happen through this -//! interface. - -use frame_support::pallet_macros::*; - -#[export_section] -mod bags_list { - use codec::{Decode, Encode, MaxEncodedLen}; - use frame_election_provider_support::ScoreProvider; - use frame_support::{ - defensive, ensure, - traits::{Defensive, DefensiveOption, Get}, - DefaultNoBound, PalletError, - }; - use scale_info::TypeInfo; - use sp_runtime::traits::{Bounded, Zero}; - use sp_std::{ - boxed::Box, - collections::{btree_map::BTreeMap, btree_set::BTreeSet}, - iter, - marker::PhantomData, - }; - - #[derive(Debug, PartialEq, Eq, Encode, Decode, MaxEncodedLen, TypeInfo, PalletError)] - pub enum ListError { - /// A duplicate id has been detected. - Duplicate, - /// An Id does not have a greater score than another Id. - NotHeavier, - /// Attempted to place node in front of a node in another bag. - NotInSameBag, - /// Given node id was not found. - NodeNotFound, - } - - /// Given a certain score, to which bag does it belong to? - /// - /// Bags are identified by their upper threshold; the value returned by this function is guaranteed - /// to be a member of `T::BagThresholds`. - /// - /// Note that even if the thresholds list does not have `T::Score::max_value()` as its final member, - /// this function behaves as if it does. - pub fn notional_bag_for, I: 'static>(score: T::Score) -> T::Score { - let thresholds = T::BagThresholds::get(); - let idx = thresholds.partition_point(|&threshold| score > threshold); - thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) - } - - /// The **ONLY** entry point of this module. All operations to the bags-list should happen through - /// this interface. It is forbidden to access other module members directly. - // - // Data structure providing efficient mostly-accurate selection of the top N id by `Score`. - // - // It's implemented as a set of linked lists. Each linked list comprises a bag of ids of - // arbitrary and unbounded length, all having a score within a particular constant range. - // This structure means that ids can be added and removed in `O(1)` time. - // - // Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While - // the users within any particular bag are sorted in an entirely arbitrary order, the overall score - // decreases as successive bags are reached. This means that it is valid to truncate - // iteration at any desired point; only those ids in the lowest bag can be excluded. This - // satisfies both the desire for fairness and the requirement for efficiency. - pub struct List, I: 'static = ()>(PhantomData<(T, I)>); - - impl, I: 'static> List { - /// Remove all data associated with the list from storage. - /// - /// ## WARNING - /// - /// this function should generally not be used in production as it could lead to a very large - /// number of storage accesses. - pub(crate) fn unsafe_clear() { - #[allow(deprecated)] - ListBags::::remove_all(None); - #[allow(deprecated)] - ListNodes::::remove_all(); - } - - /// Regenerate all of the data from the given ids. - /// - /// WARNING: this is expensive and should only ever be performed when the list needs to be - /// generated from scratch. Care needs to be taken to ensure - /// - /// This may or may not need to be called at genesis as well, based on the configuration of the - /// pallet using this `List`. - /// - /// Returns the number of ids migrated. - pub fn unsafe_regenerate( - all: impl IntoIterator, - score_of: Box T::Score>, - ) -> u32 { - // NOTE: This call is unsafe for the same reason as SortedListProvider::unsafe_regenerate. - // I.e. because it can lead to many storage accesses. - // So it is ok to call it as caller must ensure the conditions. - Self::unsafe_clear(); - Self::insert_many(all, score_of) - } - - /// Migrate the list from one set of thresholds to another. - /// - /// This should only be called as part of an intentional migration; it's fairly expensive. - /// - /// Returns the number of accounts affected. - /// - /// Preconditions: - /// - /// - `old_thresholds` is the previous list of thresholds. - /// - All `bag_upper` currently in storage are members of `old_thresholds`. - /// - `T::BagThresholds` has already been updated and is the new set of thresholds. - /// - /// Postconditions: - /// - /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. - /// - No id is changed unless required to by the difference between the old threshold list and - /// the new. - /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the new - /// threshold set. - #[allow(dead_code)] - pub fn migrate(old_thresholds: &[T::Score]) -> u32 { - let new_thresholds = T::BagThresholds::get(); - if new_thresholds == old_thresholds { - return 0 - } - - // we can't check all preconditions, but we can check one - debug_assert!( - ListBags::::iter() - .all(|(threshold, _)| old_thresholds.contains(&threshold)), - "not all `bag_upper` currently in storage are members of `old_thresholds`", - ); - debug_assert!( - ListNodes::::iter() - .all(|(_, node)| old_thresholds.contains(&node.bag_upper)), - "not all `node.bag_upper` currently in storage are members of `old_thresholds`", - ); - - let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); - let new_set: BTreeSet<_> = new_thresholds.iter().copied().collect(); - - // accounts that need to be rebagged - let mut affected_accounts = BTreeSet::new(); - // track affected old bags to make sure we only iterate them once - let mut affected_old_bags = BTreeSet::new(); - - let new_bags = new_set.difference(&old_set).copied(); - // a new bag means that all accounts previously using the old bag's threshold must now - // be rebagged - for inserted_bag in new_bags { - let affected_bag = { - // this recreates `notional_bag_for` logic, but with the old thresholds. - let idx = old_thresholds.partition_point(|&threshold| inserted_bag > threshold); - old_thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) - }; - if !affected_old_bags.insert(affected_bag) { - // If the previous threshold list was [10, 20], and we insert [3, 5], then there's - // no point iterating through bag 10 twice. - continue - } - - if let Some(bag) = Bag::::get(affected_bag) { - affected_accounts.extend(bag.iter().map(|node| node.id)); - } - } - - let removed_bags = old_set.difference(&new_set).copied(); - // a removed bag means that all members of that bag must be rebagged - for removed_bag in removed_bags.clone() { - if !affected_old_bags.insert(removed_bag) { - continue - } - - if let Some(bag) = Bag::::get(removed_bag) { - affected_accounts.extend(bag.iter().map(|node| node.id)); - } - } - - // migrate the voters whose bag has changed - let num_affected = affected_accounts.len() as u32; - let score_of = T::ScoreProvider::score; - let _removed = Self::remove_many(&affected_accounts); - debug_assert_eq!(_removed, num_affected); - let _inserted = Self::insert_many(affected_accounts.into_iter(), score_of); - debug_assert_eq!(_inserted, num_affected); - - // we couldn't previously remove the old bags because both insertion and removal assume that - // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid - // of them. - // - // it's pretty cheap to iterate this again, because both sets are in-memory and require no - // lookups. - for removed_bag in removed_bags { - debug_assert!( - !ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), - "no id should be present in a removed bag", - ); - ListBags::::remove(removed_bag); - } - - num_affected - } - - /// Returns `true` if the list contains `id`, otherwise returns `false`. - pub(crate) fn contains(id: &T::AccountId) -> bool { - ListNodes::::contains_key(id) - } - - /// Get the score of the given node, - pub fn get_score(id: &T::AccountId) -> Result { - Node::::get(id).map(|node| node.score()).ok_or(ListError::NodeNotFound) - } - - /// Iterate over all nodes in all bags in the list. - /// - /// Full iteration can be expensive; it's recommended to limit the number of items with - /// `.take(n)`. - pub(crate) fn iter() -> impl Iterator> { - // We need a touch of special handling here: because we permit `T::BagThresholds` to - // omit the final bound, we need to ensure that we explicitly include that threshold in the - // list. - // - // It's important to retain the ability to omit the final bound because it makes tests much - // easier; they can just configure `type BagThresholds = ()`. - let thresholds = T::BagThresholds::get(); - let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == - Some(&T::Score::max_value()) - { - // in the event that they included it, we can just pass the iterator through unchanged. - Box::new(iter.rev()) - } else { - // otherwise, insert it here. - Box::new(iter.chain(iter::once(T::Score::max_value())).rev()) - }; - - iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) - } - - /// Same as `iter`, but we start from a specific node. - /// - /// All items after this node are returned, excluding `start` itself. - pub(crate) fn iter_from( - start: &T::AccountId, - ) -> Result>, ListError> { - // We chain two iterators: - // 1. from the given `start` till the end of the bag - // 2. all the bags that come after `start`'s bag. - - let start_node = Node::::get(start).ok_or(ListError::NodeNotFound)?; - let start_node_upper = start_node.bag_upper; - let start_bag = sp_std::iter::successors(start_node.next(), |prev| prev.next()); - - let thresholds = T::BagThresholds::get(); - let idx = thresholds.partition_point(|&threshold| start_node_upper > threshold); - let leftover_bags = thresholds - .into_iter() - .take(idx) - .copied() - .rev() - .filter_map(Bag::get) - .flat_map(|bag| bag.iter()); - - Ok(start_bag.chain(leftover_bags)) - } - - /// Insert several ids into the appropriate bags in the list. Continues with insertions - /// if duplicates are detected. - /// - /// Returns the final count of number of ids inserted. - fn insert_many( - ids: impl IntoIterator, - score_of: impl Fn(&T::AccountId) -> T::Score, - ) -> u32 { - let mut count = 0; - ids.into_iter().for_each(|v| { - let score = score_of(&v); - if Self::insert(v, score).is_ok() { - count += 1; - } - }); - - count - } - - /// Insert a new id into the appropriate bag in the list. - /// - /// Returns an error if the list already contains `id`. - pub(crate) fn insert(id: T::AccountId, score: T::Score) -> Result<(), ListError> { - if Self::contains(&id) { - return Err(ListError::Duplicate) - } - - let bag_score = notional_bag_for::(score); - let mut bag = Bag::::get_or_make(bag_score); - // unchecked insertion is okay; we just got the correct `notional_bag_for`. - bag.insert_unchecked(id.clone(), score); - - // new inserts are always the tail, so we must write the bag. - bag.put(); - - crate::log!( - debug, - "inserted {:?} with score {:?} into bag {:?}, new count is {}", - id, - score, - bag_score, - ListNodes::::count(), - ); - - Ok(()) - } - - /// Remove an id from the list, returning an error if `id` does not exists. - pub(crate) fn remove(id: &T::AccountId) -> Result<(), ListError> { - if !Self::contains(id) { - return Err(ListError::NodeNotFound) - } - let _ = Self::remove_many(sp_std::iter::once(id)); - Ok(()) - } - - /// Remove many ids from the list. - /// - /// This is more efficient than repeated calls to `Self::remove`. - /// - /// Returns the final count of number of ids removed. - pub fn remove_many<'a>(ids: impl IntoIterator) -> u32 { - let mut bags = BTreeMap::new(); - let mut count = 0; - - for id in ids.into_iter() { - let node = match Node::::get(id) { - Some(node) => node, - None => continue, - }; - count += 1; - - if !node.is_terminal() { - // this node is not a head or a tail and thus the bag does not need to be updated - node.excise() - } else { - // this node is a head or tail, so the bag needs to be updated - let bag = bags - .entry(node.bag_upper) - .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); - // node.bag_upper must be correct, therefore this bag will contain this node. - bag.remove_node_unchecked(&node); - } - - // now get rid of the node itself - node.remove_from_storage_unchecked() - } - - for (_, bag) in bags { - bag.put(); - } - - count - } - - /// Update a node's position in the list. - /// - /// If the node was in the correct bag, no effect. If the node was in the incorrect bag, they - /// are moved into the correct bag. - /// - /// Returns `Some((old_idx, new_idx))` if the node moved, otherwise `None`. In both cases, the - /// node's score is written to the `score` field. Thus, this is not a noop, even if `None`. - /// - /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by - /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient - /// to call [`self.remove_many`] followed by [`self.insert_many`]. - pub(crate) fn update_position_for( - mut node: Node, - new_score: T::Score, - ) -> Option<(T::Score, T::Score)> { - node.score = new_score; - if node.is_misplaced(new_score) { - let old_bag_upper = node.bag_upper; - - if !node.is_terminal() { - // this node is not a head or a tail, so we can just cut it out of the list. update - // and put the prev and next of this node, we do `node.put` inside `insert_note`. - node.excise(); - } else if let Some(mut bag) = Bag::::get(node.bag_upper) { - // this is a head or tail, so the bag must be updated. - bag.remove_node_unchecked(&node); - bag.put(); - } else { - frame_support::defensive!( - "Node did not have a bag; BagsList is in an inconsistent state" - ); - } - - // put the node into the appropriate new bag. - let new_bag_upper = notional_bag_for::(new_score); - let mut bag = Bag::::get_or_make(new_bag_upper); - // prev, next, and bag_upper of the node are updated inside `insert_node`, also - // `node.put` is in there. - bag.insert_node_unchecked(node); - bag.put(); - - Some((old_bag_upper, new_bag_upper)) - } else { - // just write the new score. - node.put(); - None - } - } - - /// Put `heavier_id` to the position directly in front of `lighter_id`. Both ids must be in the - /// same bag and the `score_of` `lighter_id` must be less than that of `heavier_id`. - pub(crate) fn put_in_front_of( - lighter_id: &T::AccountId, - heavier_id: &T::AccountId, - ) -> Result<(), ListError> { - let lighter_node = Node::::get(&lighter_id).ok_or(ListError::NodeNotFound)?; - let heavier_node = Node::::get(&heavier_id).ok_or(ListError::NodeNotFound)?; - - ensure!(lighter_node.bag_upper == heavier_node.bag_upper, ListError::NotInSameBag); - - // this is the most expensive check, so we do it last. - ensure!( - T::ScoreProvider::score(&heavier_id) > T::ScoreProvider::score(&lighter_id), - ListError::NotHeavier - ); - - // remove the heavier node from this list. Note that this removes the node from storage and - // decrements the node counter. - let _ = - Self::remove(&heavier_id).defensive_proof("both nodes have been checked to exist; qed"); - - // re-fetch `lighter_node` from storage since it may have been updated when `heavier_node` - // was removed. - let lighter_node = - Node::::get(lighter_id).defensive_ok_or_else(|| ListError::NodeNotFound)?; - - // insert `heavier_node` directly in front of `lighter_node`. This will update both nodes - // in storage and update the node counter. - Self::insert_at_unchecked(lighter_node, heavier_node); - - Ok(()) - } - - /// Insert `node` directly in front of `at`. - /// - /// WARNINGS: - /// - this is a naive function in that it does not check if `node` belongs to the same bag as - /// `at`. It is expected that the call site will check preconditions. - /// - this will panic if `at.bag_upper` is not a bag that already exists in storage. - pub fn insert_at_unchecked(mut at: Node, mut node: Node) { - // connect `node` to its new `prev`. - node.prev = at.prev.clone(); - if let Some(mut prev) = at.prev() { - prev.next = Some(node.id().clone()); - prev.put() - } - - // connect `node` and `at`. - node.next = Some(at.id().clone()); - at.prev = Some(node.id().clone()); - - if node.is_terminal() { - // `node` is the new head, so we make sure the bag is updated. Note, - // since `node` is always in front of `at` we know that 1) there is always at least 2 - // nodes in the bag, and 2) only `node` could be the head and only `at` could be the - // tail. - let mut bag = Bag::::get(at.bag_upper) - .expect("given nodes must always have a valid bag. qed."); - - if node.prev == None { - bag.head = Some(node.id().clone()) - } - - bag.put() - }; - - // write the updated nodes to storage. - at.put(); - node.put(); - } - - /// Check the internal state of the list. - /// - /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) - /// is being used, after all other staking data (such as counter) has been updated. It checks: - /// - /// * there are no duplicate ids, - /// * length of this list is in sync with `ListNodes::count()`, - /// * and sanity-checks all bags and nodes. This will cascade down all the checks and makes sure - /// all bags and nodes are checked per *any* update to `List`. - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - pub(crate) fn do_try_state() -> Result<(), &'static str> { - let mut seen_in_list = BTreeSet::new(); - ensure!( - Self::iter().map(|node| node.id).all(|id| seen_in_list.insert(id)), - "duplicate identified", - ); - - let iter_count = Self::iter().count() as u32; - let stored_count = ListNodes::::count(); - let nodes_count = ListNodes::::iter().count() as u32; - ensure!(iter_count == stored_count, "iter_count != stored_count"); - ensure!(stored_count == nodes_count, "stored_count != nodes_count"); - - crate::log!(trace, "count of nodes: {}", stored_count); - - let active_bags = { - let thresholds = T::BagThresholds::get().iter().copied(); - let thresholds: Vec = - if thresholds.clone().last() == Some(T::Score::max_value()) { - // in the event that they included it, we don't need to make any changes - thresholds.collect() - } else { - // otherwise, insert it here. - thresholds.chain(iter::once(T::Score::max_value())).collect() - }; - thresholds.into_iter().filter_map(|t| Bag::::get(t)) - }; - - let _ = active_bags.clone().try_for_each(|b| b.do_try_state())?; - - let nodes_in_bags_count = - active_bags.clone().fold(0u32, |acc, cur| acc + cur.iter().count() as u32); - ensure!(nodes_count == nodes_in_bags_count, "stored_count != nodes_in_bags_count"); - - crate::log!(trace, "count of active bags {}", active_bags.count()); - - // check that all nodes are sane. We check the `ListNodes` storage item directly in case we - // have some "stale" nodes that are not in a bag. - for (_id, node) in ListNodes::::iter() { - node.do_try_state()? - } - - Ok(()) - } - - /// Returns the nodes of all non-empty bags. For testing and benchmarks. - #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] - #[allow(dead_code)] - pub(crate) fn get_bags() -> Vec<(T::Score, Vec)> { - use frame_support::traits::Get as _; - - let thresholds = T::BagThresholds::get(); - let iter = thresholds.iter().copied(); - let iter: Box> = if thresholds.last() == - Some(&T::Score::max_value()) - { - // in the event that they included it, we can just pass the iterator through unchanged. - Box::new(iter) - } else { - // otherwise, insert it here. - Box::new(iter.chain(sp_std::iter::once(T::Score::max_value()))) - }; - - iter.filter_map(|t| { - Bag::::get(t) - .map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) - }) - .collect::>() - } - - pub fn count() -> u32 { - ListNodes::::count() - } - - pub fn bag_count() -> usize { - ListBags::::iter().count() - } - - #[cfg(test)] - pub fn insert_unchecked(account_id: T::AccountId, node: Node) { - ListNodes::::insert(account_id, node); - } - } - - /// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. - #[derive(Encode, Decode, MaxEncodedLen, TypeInfo)] - #[codec(mel_bound())] - #[scale_info(skip_type_params(T, I))] - #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] - pub struct Node, I: 'static = ()> { - pub(crate) id: T::AccountId, - pub(crate) prev: Option, - pub(crate) next: Option, - pub(crate) bag_upper: T::Score, - pub(crate) score: T::Score, - #[codec(skip)] - pub(crate) _phantom: PhantomData, - } - - impl, I: 'static> Node { - /// Get a node by id. - pub fn get(id: &T::AccountId) -> Option> { - ListNodes::::try_get(id).ok() - } - - /// Put the node back into storage. - pub fn put(self) { - ListNodes::::insert(self.id.clone(), self); - } - - /// Update neighboring nodes to point to reach other. - /// - /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call - /// `self.put`. - fn excise(&self) { - // Update previous node. - if let Some(mut prev) = self.prev() { - prev.next = self.next.clone(); - prev.put(); - } - // Update next self. - if let Some(mut next) = self.next() { - next.prev = self.prev.clone(); - next.put(); - } - } - - /// This is a naive function that removes a node from the `ListNodes` storage item. - /// - /// It is naive because it does not check if the node has first been removed from its bag. - fn remove_from_storage_unchecked(&self) { - ListNodes::::remove(&self.id) - } - - /// Get the previous node in the bag. - fn prev(&self) -> Option> { - self.prev.as_ref().and_then(|id| Node::get(id)) - } - - /// Get the next node in the bag. - fn next(&self) -> Option> { - self.next.as_ref().and_then(|id| Node::get(id)) - } - - /// `true` when this voter is in the wrong bag. - pub fn is_misplaced(&self, current_score: T::Score) -> bool { - notional_bag_for::(current_score) != self.bag_upper - } - - /// `true` when this voter is a bag head or tail. - fn is_terminal(&self) -> bool { - self.prev.is_none() || self.next.is_none() - } - - /// Get the underlying voter. - pub(crate) fn id(&self) -> &T::AccountId { - &self.id - } - - /// Get the current vote weight of the node. - pub(crate) fn score(&self) -> T::Score { - self.score - } - - /// Get the underlying voter (public fo tests). - #[cfg(feature = "std")] - #[allow(dead_code)] - pub fn std_id(&self) -> &T::AccountId { - &self.id - } - - #[cfg(any(feature = "runtime-benchmarks", feature = "fuzz", test))] - pub fn set_score(&mut self, s: T::Score) { - self.score = s - } - - /// The bag this nodes belongs to (public for benchmarks). - #[cfg(feature = "runtime-benchmarks")] - #[allow(dead_code)] - pub fn bag_upper(&self) -> T::Score { - self.bag_upper - } - - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - pub fn do_try_state(&self) -> Result<(), &'static str> { - let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; - - let id = self.id(); - - frame_support::ensure!( - expected_bag.contains(id), - "node does not exist in the expected bag" - ); - - let non_terminal_check = !self.is_terminal() && - expected_bag.head.as_ref() != Some(id) && - expected_bag.tail.as_ref() != Some(id); - let terminal_check = - expected_bag.head.as_ref() == Some(id) || expected_bag.tail.as_ref() == Some(id); - frame_support::ensure!( - non_terminal_check || terminal_check, - "a terminal node is neither its bag head or tail" - ); - - Ok(()) - } - } - - /// A Bag is a doubly-linked list of ids, where each id is mapped to a [`Node`]. - /// - /// Note that we maintain both head and tail pointers. While it would be possible to get away with - /// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more - /// desirable to ensure that there is some element of first-come, first-serve to the list's - /// iteration so that there's no incentive to churn ids positioning to improve the chances of - /// appearing within the ids set. - #[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen, TypeInfo)] - #[codec(mel_bound())] - #[scale_info(skip_type_params(T, I))] - #[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] - pub struct Bag, I: 'static = ()> { - pub head: Option, - pub tail: Option, - - #[codec(skip)] - pub bag_upper: T::Score, - #[codec(skip)] - pub _phantom: PhantomData, - } - - impl, I: 'static> Bag { - #[cfg(test)] - pub(crate) fn new( - head: Option, - tail: Option, - bag_upper: T::Score, - ) -> Self { - Self { head, tail, bag_upper, _phantom: PhantomData } - } - - /// Get a bag by its upper score. - pub(crate) fn get(bag_upper: T::Score) -> Option> { - ListBags::::try_get(bag_upper).ok().map(|mut bag| { - bag.bag_upper = bag_upper; - bag - }) - } - - pub fn get_raw(score: T::Score) -> Option> { - ListBags::::get(score) - } - - /// Get a bag by its upper score or make it, appropriately initialized. Does not check if - /// if `bag_upper` is a valid threshold. - pub fn get_or_make(bag_upper: T::Score) -> Bag { - Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) - } - - /// `True` if self is empty. - fn is_empty(&self) -> bool { - self.head.is_none() && self.tail.is_none() - } - - /// Put the bag back into storage. - pub fn put(self) { - if self.is_empty() { - ListBags::::remove(self.bag_upper); - } else { - ListBags::::insert(self.bag_upper, self); - } - } - - /// Get the head node in this bag. - fn head(&self) -> Option> { - self.head.as_ref().and_then(|id| Node::get(id)) - } - - /// Get the tail node in this bag. - fn tail(&self) -> Option> { - self.tail.as_ref().and_then(|id| Node::get(id)) - } - - /// Iterate over the nodes in this bag. - pub(crate) fn iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) - } - - /// Insert a new id into this bag. - /// - /// This is private on purpose because it's naive: it doesn't check whether this is the - /// appropriate bag for this id at all. Generally, use [`List::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the nodes. You still need to call - /// `self.put()` after use. - pub fn insert_unchecked(&mut self, id: T::AccountId, score: T::Score) { - // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long - // as this bag is the correct one, we're good. All calls to this must come after getting the - // correct [`notional_bag_for`]. - self.insert_node_unchecked(Node:: { - id, - prev: None, - next: None, - bag_upper: Zero::zero(), - score, - _phantom: PhantomData, - }); - } - - /// Insert a node into this bag. - /// - /// This is private on purpose because it's naive; it doesn't check whether this is the - /// appropriate bag for this node at all. Generally, use [`List::insert`] instead. - /// - /// Storage note: this modifies storage, but only for the node. You still need to call - /// `self.put()` after use. - pub fn insert_node_unchecked(&mut self, mut node: Node) { - if let Some(tail) = &self.tail { - if *tail == node.id { - // this should never happen, but this check prevents one path to a worst case - // infinite loop. - defensive!("system logic error: inserting a node who has the id of tail"); - return - }; - } - - // re-set the `bag_upper`. Regardless of whatever the node had previously, now it is going - // to be `self.bag_upper`. - node.bag_upper = self.bag_upper; - - let id = node.id.clone(); - // update this node now, treating it as the new tail. - node.prev = self.tail.clone(); - node.next = None; - node.put(); - - // update the previous tail. - if let Some(mut old_tail) = self.tail() { - old_tail.next = Some(id.clone()); - old_tail.put(); - } - self.tail = Some(id.clone()); - - // ensure head exist. This is only set when the length of the bag is just 1, i.e. if this is - // the first insertion into the bag. In this case, both head and tail should point to the - // same node. - if self.head.is_none() { - self.head = Some(id); - debug_assert!(self.iter().count() == 1); - } - } - - /// Remove a node from this bag. - /// - /// This is private on purpose because it doesn't check whether this bag contains the node in - /// the first place. Generally, use [`List::remove`] instead, similar to `insert_unchecked`. - /// - /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call - /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. - pub fn remove_node_unchecked(&mut self, node: &Node) { - // reassign neighboring nodes. - node.excise(); - - // clear the bag head/tail pointers as necessary. - if self.tail.as_ref() == Some(&node.id) { - self.tail = node.prev.clone(); - } - if self.head.as_ref() == Some(&node.id) { - self.head = node.next.clone(); - } - } - - /// Check the internal state of the bag. - /// - /// Should be called by the call-site, after any mutating operation on a bag. The call site of - /// this struct is always `List`. - /// - /// * Ensures head has no prev. - /// * Ensures tail has no next. - /// * Ensures there are no loops, traversal from head to tail is correct. - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - pub fn do_try_state(&self) -> Result<(), &'static str> { - frame_support::ensure!( - self.head() - .map(|head| head.prev().is_none()) - // if there is no head, then there must not be a tail, meaning that the bag is - // empty. - .unwrap_or_else(|| self.tail.is_none()), - "head has a prev" - ); - - frame_support::ensure!( - self.tail() - .map(|tail| tail.next().is_none()) - // if there is no tail, then there must not be a head, meaning that the bag is - // empty. - .unwrap_or_else(|| self.head.is_none()), - "tail has a next" - ); - - let mut seen_in_bag = BTreeSet::new(); - frame_support::ensure!( - self.iter() - .map(|node| node.id) - // each voter is only seen once, thus there is no cycle within a bag - .all(|voter| seen_in_bag.insert(voter)), - "duplicate found in bag" - ); - - Ok(()) - } - - /// Iterate over the nodes in this bag (public for tests). - #[cfg(feature = "std")] - #[allow(dead_code)] - pub fn std_iter(&self) -> impl Iterator> { - sp_std::iter::successors(self.head(), |prev| prev.next()) - } - - /// Check if the bag contains a node with `id`. - #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] - fn contains(&self, id: &T::AccountId) -> bool { - self.iter().any(|n| n.id() == id) - } - } - - /// A single node, within some bag. - /// - /// Nodes store links forward and back within their respective bags. - #[pallet::storage] - type ListNodes, I: 'static = ()> = - CountedStorageMap<_, Twox64Concat, T::AccountId, Node>; - - /// A bag stored in storage. - /// - /// Stores a `Bag` struct, which stores head and tail pointers to itself. - #[pallet::storage] - type ListBags, I: 'static = ()> = - StorageMap<_, Twox64Concat, T::Score, Bag>; -} \ No newline at end of file diff --git a/frame/bags-list/Cargo.toml b/frame/balances/bags-list/Cargo.toml similarity index 95% rename from frame/bags-list/Cargo.toml rename to frame/balances/bags-list/Cargo.toml index 78b49a9993c03..1678ce1ba2ac6 100644 --- a/frame/bags-list/Cargo.toml +++ b/frame/balances/bags-list/Cargo.toml @@ -15,7 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"] # parity codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false, features = ["derive"] } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -bags-list = { version = "1.0.0", default-features = false, path = "./src/bags-list" } # primitives sp-runtime = { version = "7.0.0", default-features = false, path = "../../primitives/runtime" } @@ -28,7 +27,6 @@ frame-election-provider-support = { version = "4.0.0-dev", default-features = fa # third party log = { version = "0.4.17", default-features = false } -macro_magic = "0.3.1" # Optional imports for benchmarking frame-benchmarking = { version = "4.0.0-dev", path = "../benchmarking", optional = true, default-features = false } @@ -61,7 +59,6 @@ std = [ "frame-system/std", "frame-election-provider-support/std", "log/std", - "bags-list/std" ] runtime-benchmarks = [ "frame-benchmarking/runtime-benchmarks", diff --git a/frame/bags-list/fuzzer/.gitignore b/frame/balances/bags-list/fuzzer/.gitignore similarity index 100% rename from frame/bags-list/fuzzer/.gitignore rename to frame/balances/bags-list/fuzzer/.gitignore diff --git a/frame/bags-list/fuzzer/Cargo.toml b/frame/balances/bags-list/fuzzer/Cargo.toml similarity index 100% rename from frame/bags-list/fuzzer/Cargo.toml rename to frame/balances/bags-list/fuzzer/Cargo.toml diff --git a/frame/bags-list/fuzzer/src/main.rs b/frame/balances/bags-list/fuzzer/src/main.rs similarity index 100% rename from frame/bags-list/fuzzer/src/main.rs rename to frame/balances/bags-list/fuzzer/src/main.rs diff --git a/frame/bags-list/remote-tests/Cargo.toml b/frame/balances/bags-list/remote-tests/Cargo.toml similarity index 100% rename from frame/bags-list/remote-tests/Cargo.toml rename to frame/balances/bags-list/remote-tests/Cargo.toml diff --git a/frame/bags-list/remote-tests/src/lib.rs b/frame/balances/bags-list/remote-tests/src/lib.rs similarity index 100% rename from frame/bags-list/remote-tests/src/lib.rs rename to frame/balances/bags-list/remote-tests/src/lib.rs diff --git a/frame/bags-list/remote-tests/src/migration.rs b/frame/balances/bags-list/remote-tests/src/migration.rs similarity index 100% rename from frame/bags-list/remote-tests/src/migration.rs rename to frame/balances/bags-list/remote-tests/src/migration.rs diff --git a/frame/bags-list/remote-tests/src/snapshot.rs b/frame/balances/bags-list/remote-tests/src/snapshot.rs similarity index 100% rename from frame/bags-list/remote-tests/src/snapshot.rs rename to frame/balances/bags-list/remote-tests/src/snapshot.rs diff --git a/frame/bags-list/remote-tests/src/try_state.rs b/frame/balances/bags-list/remote-tests/src/try_state.rs similarity index 100% rename from frame/bags-list/remote-tests/src/try_state.rs rename to frame/balances/bags-list/remote-tests/src/try_state.rs diff --git a/frame/bags-list/src/benchmarks.rs b/frame/balances/bags-list/src/benchmarks.rs similarity index 99% rename from frame/bags-list/src/benchmarks.rs rename to frame/balances/bags-list/src/benchmarks.rs index 2dae68452a9d2..0c3955c0d7b79 100644 --- a/frame/bags-list/src/benchmarks.rs +++ b/frame/balances/bags-list/src/benchmarks.rs @@ -18,7 +18,7 @@ //! Benchmarks for the bags list pallet. use super::*; -use crate::List; +use crate::list::List; use frame_benchmarking::v1::{ account, benchmarks_instance_pallet, whitelist_account, whitelisted_caller, }; diff --git a/frame/bags-list/src/lib.rs b/frame/balances/bags-list/src/lib.rs similarity index 89% rename from frame/bags-list/src/lib.rs rename to frame/balances/bags-list/src/lib.rs index c7dcc05a9e79e..156c52cc87c45 100644 --- a/frame/bags-list/src/lib.rs +++ b/frame/balances/bags-list/src/lib.rs @@ -56,22 +56,24 @@ use codec::FullCodec; use frame_election_provider_support::{ScoreProvider, SortedListProvider}; use frame_system::ensure_signed; -use sp_runtime::traits::{AtLeast32BitUnsigned, StaticLookup}; +use sp_runtime::traits::{AtLeast32BitUnsigned, Bounded, StaticLookup}; use sp_std::prelude::*; -use bags_list::*; + +#[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] +use sp_runtime::TryRuntimeError; #[cfg(any(feature = "runtime-benchmarks", test))] mod benchmarks; +mod list; pub mod migrations; #[cfg(any(test, feature = "fuzz"))] pub mod mock; #[cfg(test)] mod tests; -#[cfg(test)] -mod tests_bags_list; pub mod weights; +pub use list::{notional_bag_for, Bag, List, ListError, Node}; pub use pallet::*; pub use weights::WeightInfo; @@ -93,9 +95,6 @@ macro_rules! log { type AccountIdLookupOf = <::Lookup as StaticLookup>::Source; -use frame_support::pallet_macros::*; - -#[import_section(bags_list)] #[frame_support::pallet] pub mod pallet { use super::*; @@ -179,6 +178,20 @@ pub mod pallet { + MaxEncodedLen; } + /// A single node, within some bag. + /// + /// Nodes store links forward and back within their respective bags. + #[pallet::storage] + pub(crate) type ListNodes, I: 'static = ()> = + CountedStorageMap<_, Twox64Concat, T::AccountId, list::Node>; + + /// A bag stored in storage. + /// + /// Stores a `Bag` struct, which stores head and tail pointers to itself. + #[pallet::storage] + pub(crate) type ListBags, I: 'static = ()> = + StorageMap<_, Twox64Concat, T::Score, list::Bag>; + #[pallet::event] #[pallet::generate_deposit(pub(crate) fn deposit_event)] pub enum Event, I: 'static = ()> { @@ -250,18 +263,14 @@ pub mod pallet { impl, I: 'static> Hooks> for Pallet { fn integrity_test() { // ensure they are strictly increasing, this also implies that duplicates are detected. - for window in T::BagThresholds::get().windows(2) { - assert!(window[1] > window[0], "thresholds must strictly increase, and have no duplicates"); - } - // FAIL-CI: Following snippet fails under `import_section` but above snippet works - // assert!( - // T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), - // "thresholds must strictly increase, and have no duplicates", - // ); + assert!( + T::BagThresholds::get().windows(2).all(|window| window[1] > window[0]), + "thresholds must strictly increase, and have no duplicates", + ); } #[cfg(feature = "try-runtime")] - fn try_state(_: BlockNumberFor) -> Result<(), &'static str> { + fn try_state(_: BlockNumberFor) -> Result<(), TryRuntimeError> { >::try_state() } } @@ -269,7 +278,7 @@ pub mod pallet { #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] impl, I: 'static> Pallet { - pub fn do_try_state() -> Result<(), &'static str> { + pub fn do_try_state() -> Result<(), TryRuntimeError> { List::::do_try_state() } } @@ -283,7 +292,7 @@ impl, I: 'static> Pallet { new_score: T::Score, ) -> Result, ListError> { // If no voter at that node, don't do anything. the caller just wasted the fee to call this. - let node = Node::::get(&account).ok_or(ListError::NodeNotFound)?; + let node = list::Node::::get(&account).ok_or(ListError::NodeNotFound)?; let maybe_movement = List::update_position_for(node, new_score); if let Some((from, to)) = maybe_movement { Self::deposit_event(Event::::Rebagged { who: account.clone(), from, to }); @@ -292,7 +301,11 @@ impl, I: 'static> Pallet { Ok(maybe_movement) } - + /// Equivalent to `ListBags::get`, but public. Useful for tests in outside of this crate. + #[cfg(feature = "std")] + pub fn list_bags_get(score: T::Score) -> Option> { + ListBags::get(score) + } } impl, I: 'static> SortedListProvider for Pallet { @@ -311,7 +324,7 @@ impl, I: 'static> SortedListProvider for Pallet } fn count() -> u32 { - List::::count() + ListNodes::::count() } fn contains(id: &T::AccountId) -> bool { @@ -345,7 +358,7 @@ impl, I: 'static> SortedListProvider for Pallet } #[cfg(feature = "try-runtime")] - fn try_state() -> Result<(), &'static str> { + fn try_state() -> Result<(), TryRuntimeError> { Self::do_try_state() } @@ -360,7 +373,7 @@ impl, I: 'static> SortedListProvider for Pallet fn score_update_worst_case(who: &T::AccountId, is_increase: bool) -> Self::Score { use frame_support::traits::Get as _; let thresholds = T::BagThresholds::get(); - let node = Node::::get(who).unwrap(); + let node = list::Node::::get(who).unwrap(); let current_bag_idx = thresholds .iter() .chain(sp_std::iter::once(&T::Score::max_value())) @@ -387,7 +400,7 @@ impl, I: 'static> ScoreProvider for Pallet { Node::::get(id).map(|node| node.score()).unwrap_or_default() } - frame_election_provider_support::runtime_benchmarks_or_fuzz_enabled! { + frame_election_provider_support::runtime_benchmarks_fuzz_or_std_enabled! { fn set_score_of(id: &T::AccountId, new_score: T::Score) { ListNodes::::mutate(id, |maybe_node| { if let Some(node) = maybe_node.as_mut() { @@ -398,4 +411,4 @@ impl, I: 'static> ScoreProvider for Pallet { }) } } -} \ No newline at end of file +} diff --git a/frame/balances/bags-list/src/list/mod.rs b/frame/balances/bags-list/src/list/mod.rs new file mode 100644 index 0000000000000..d8626080e2523 --- /dev/null +++ b/frame/balances/bags-list/src/list/mod.rs @@ -0,0 +1,920 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation of a "bags list": a semi-sorted list where ordering granularity is dictated by +//! configurable thresholds that delineate the boundaries of bags. It uses a pattern of composite +//! data structures, where multiple storage items are masked by one outer API. See +//! [`crate::ListNodes`], [`crate::ListBags`] for more information. +//! +//! The outer API of this module is the [`List`] struct. It wraps all acceptable operations on top +//! of the aggregate linked list. All operations with the bags list should happen through this +//! interface. + +use crate::Config; +use codec::{Decode, Encode, MaxEncodedLen}; +use frame_election_provider_support::ScoreProvider; +use frame_support::{ + defensive, ensure, + traits::{Defensive, DefensiveOption, Get}, + DefaultNoBound, PalletError, +}; +use scale_info::TypeInfo; +use sp_runtime::traits::{Bounded, Zero}; +use sp_std::{ + boxed::Box, + collections::{btree_map::BTreeMap, btree_set::BTreeSet}, + iter, + marker::PhantomData, + prelude::*, +}; + +#[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] +use sp_runtime::TryRuntimeError; + +#[derive(Debug, PartialEq, Eq, Encode, Decode, MaxEncodedLen, TypeInfo, PalletError)] +pub enum ListError { + /// A duplicate id has been detected. + Duplicate, + /// An Id does not have a greater score than another Id. + NotHeavier, + /// Attempted to place node in front of a node in another bag. + NotInSameBag, + /// Given node id was not found. + NodeNotFound, +} + +#[cfg(test)] +mod tests; + +/// Given a certain score, to which bag does it belong to? +/// +/// Bags are identified by their upper threshold; the value returned by this function is guaranteed +/// to be a member of `T::BagThresholds`. +/// +/// Note that even if the thresholds list does not have `T::Score::max_value()` as its final member, +/// this function behaves as if it does. +pub fn notional_bag_for, I: 'static>(score: T::Score) -> T::Score { + let thresholds = T::BagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| score > threshold); + thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) +} + +/// The **ONLY** entry point of this module. All operations to the bags-list should happen through +/// this interface. It is forbidden to access other module members directly. +// +// Data structure providing efficient mostly-accurate selection of the top N id by `Score`. +// +// It's implemented as a set of linked lists. Each linked list comprises a bag of ids of +// arbitrary and unbounded length, all having a score within a particular constant range. +// This structure means that ids can be added and removed in `O(1)` time. +// +// Iteration is accomplished by chaining the iteration of each bag, from greatest to least. While +// the users within any particular bag are sorted in an entirely arbitrary order, the overall score +// decreases as successive bags are reached. This means that it is valid to truncate +// iteration at any desired point; only those ids in the lowest bag can be excluded. This +// satisfies both the desire for fairness and the requirement for efficiency. +pub struct List, I: 'static = ()>(PhantomData<(T, I)>); + +impl, I: 'static> List { + /// Remove all data associated with the list from storage. + /// + /// ## WARNING + /// + /// this function should generally not be used in production as it could lead to a very large + /// number of storage accesses. + pub(crate) fn unsafe_clear() { + #[allow(deprecated)] + crate::ListBags::::remove_all(None); + #[allow(deprecated)] + crate::ListNodes::::remove_all(); + } + + /// Regenerate all of the data from the given ids. + /// + /// WARNING: this is expensive and should only ever be performed when the list needs to be + /// generated from scratch. Care needs to be taken to ensure + /// + /// This may or may not need to be called at genesis as well, based on the configuration of the + /// pallet using this `List`. + /// + /// Returns the number of ids migrated. + pub fn unsafe_regenerate( + all: impl IntoIterator, + score_of: Box T::Score>, + ) -> u32 { + // NOTE: This call is unsafe for the same reason as SortedListProvider::unsafe_regenerate. + // I.e. because it can lead to many storage accesses. + // So it is ok to call it as caller must ensure the conditions. + Self::unsafe_clear(); + Self::insert_many(all, score_of) + } + + /// Migrate the list from one set of thresholds to another. + /// + /// This should only be called as part of an intentional migration; it's fairly expensive. + /// + /// Returns the number of accounts affected. + /// + /// Preconditions: + /// + /// - `old_thresholds` is the previous list of thresholds. + /// - All `bag_upper` currently in storage are members of `old_thresholds`. + /// - `T::BagThresholds` has already been updated and is the new set of thresholds. + /// + /// Postconditions: + /// + /// - All `bag_upper` currently in storage are members of `T::BagThresholds`. + /// - No id is changed unless required to by the difference between the old threshold list and + /// the new. + /// - ids whose bags change at all are implicitly rebagged into the appropriate bag in the new + /// threshold set. + #[allow(dead_code)] + pub fn migrate(old_thresholds: &[T::Score]) -> u32 { + let new_thresholds = T::BagThresholds::get(); + if new_thresholds == old_thresholds { + return 0 + } + + // we can't check all preconditions, but we can check one + debug_assert!( + crate::ListBags::::iter() + .all(|(threshold, _)| old_thresholds.contains(&threshold)), + "not all `bag_upper` currently in storage are members of `old_thresholds`", + ); + debug_assert!( + crate::ListNodes::::iter() + .all(|(_, node)| old_thresholds.contains(&node.bag_upper)), + "not all `node.bag_upper` currently in storage are members of `old_thresholds`", + ); + + let old_set: BTreeSet<_> = old_thresholds.iter().copied().collect(); + let new_set: BTreeSet<_> = new_thresholds.iter().copied().collect(); + + // accounts that need to be rebagged + let mut affected_accounts = BTreeSet::new(); + // track affected old bags to make sure we only iterate them once + let mut affected_old_bags = BTreeSet::new(); + + let new_bags = new_set.difference(&old_set).copied(); + // a new bag means that all accounts previously using the old bag's threshold must now + // be rebagged + for inserted_bag in new_bags { + let affected_bag = { + // this recreates `notional_bag_for` logic, but with the old thresholds. + let idx = old_thresholds.partition_point(|&threshold| inserted_bag > threshold); + old_thresholds.get(idx).copied().unwrap_or_else(T::Score::max_value) + }; + if !affected_old_bags.insert(affected_bag) { + // If the previous threshold list was [10, 20], and we insert [3, 5], then there's + // no point iterating through bag 10 twice. + continue + } + + if let Some(bag) = Bag::::get(affected_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } + } + + let removed_bags = old_set.difference(&new_set).copied(); + // a removed bag means that all members of that bag must be rebagged + for removed_bag in removed_bags.clone() { + if !affected_old_bags.insert(removed_bag) { + continue + } + + if let Some(bag) = Bag::::get(removed_bag) { + affected_accounts.extend(bag.iter().map(|node| node.id)); + } + } + + // migrate the voters whose bag has changed + let num_affected = affected_accounts.len() as u32; + let score_of = T::ScoreProvider::score; + let _removed = Self::remove_many(&affected_accounts); + debug_assert_eq!(_removed, num_affected); + let _inserted = Self::insert_many(affected_accounts.into_iter(), score_of); + debug_assert_eq!(_inserted, num_affected); + + // we couldn't previously remove the old bags because both insertion and removal assume that + // it's always safe to add a bag if it's not present. Now that that's sorted, we can get rid + // of them. + // + // it's pretty cheap to iterate this again, because both sets are in-memory and require no + // lookups. + for removed_bag in removed_bags { + debug_assert!( + !crate::ListNodes::::iter().any(|(_, node)| node.bag_upper == removed_bag), + "no id should be present in a removed bag", + ); + crate::ListBags::::remove(removed_bag); + } + + num_affected + } + + /// Returns `true` if the list contains `id`, otherwise returns `false`. + pub(crate) fn contains(id: &T::AccountId) -> bool { + crate::ListNodes::::contains_key(id) + } + + /// Get the score of the given node, + pub fn get_score(id: &T::AccountId) -> Result { + Node::::get(id).map(|node| node.score()).ok_or(ListError::NodeNotFound) + } + + /// Iterate over all nodes in all bags in the list. + /// + /// Full iteration can be expensive; it's recommended to limit the number of items with + /// `.take(n)`. + pub(crate) fn iter() -> impl Iterator> { + // We need a touch of special handling here: because we permit `T::BagThresholds` to + // omit the final bound, we need to ensure that we explicitly include that threshold in the + // list. + // + // It's important to retain the ability to omit the final bound because it makes tests much + // easier; they can just configure `type BagThresholds = ()`. + let thresholds = T::BagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == + Some(&T::Score::max_value()) + { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter.rev()) + } else { + // otherwise, insert it here. + Box::new(iter.chain(iter::once(T::Score::max_value())).rev()) + }; + + iter.filter_map(Bag::get).flat_map(|bag| bag.iter()) + } + + /// Same as `iter`, but we start from a specific node. + /// + /// All items after this node are returned, excluding `start` itself. + pub(crate) fn iter_from( + start: &T::AccountId, + ) -> Result>, ListError> { + // We chain two iterators: + // 1. from the given `start` till the end of the bag + // 2. all the bags that come after `start`'s bag. + + let start_node = Node::::get(start).ok_or(ListError::NodeNotFound)?; + let start_node_upper = start_node.bag_upper; + let start_bag = sp_std::iter::successors(start_node.next(), |prev| prev.next()); + + let thresholds = T::BagThresholds::get(); + let idx = thresholds.partition_point(|&threshold| start_node_upper > threshold); + let leftover_bags = thresholds + .into_iter() + .take(idx) + .copied() + .rev() + .filter_map(Bag::get) + .flat_map(|bag| bag.iter()); + + Ok(start_bag.chain(leftover_bags)) + } + + /// Insert several ids into the appropriate bags in the list. Continues with insertions + /// if duplicates are detected. + /// + /// Returns the final count of number of ids inserted. + fn insert_many( + ids: impl IntoIterator, + score_of: impl Fn(&T::AccountId) -> T::Score, + ) -> u32 { + let mut count = 0; + ids.into_iter().for_each(|v| { + let score = score_of(&v); + if Self::insert(v, score).is_ok() { + count += 1; + } + }); + + count + } + + /// Insert a new id into the appropriate bag in the list. + /// + /// Returns an error if the list already contains `id`. + pub(crate) fn insert(id: T::AccountId, score: T::Score) -> Result<(), ListError> { + if Self::contains(&id) { + return Err(ListError::Duplicate) + } + + let bag_score = notional_bag_for::(score); + let mut bag = Bag::::get_or_make(bag_score); + // unchecked insertion is okay; we just got the correct `notional_bag_for`. + bag.insert_unchecked(id.clone(), score); + + // new inserts are always the tail, so we must write the bag. + bag.put(); + + crate::log!( + debug, + "inserted {:?} with score {:?} into bag {:?}, new count is {}", + id, + score, + bag_score, + crate::ListNodes::::count(), + ); + + Ok(()) + } + + /// Remove an id from the list, returning an error if `id` does not exists. + pub(crate) fn remove(id: &T::AccountId) -> Result<(), ListError> { + if !Self::contains(id) { + return Err(ListError::NodeNotFound) + } + let _ = Self::remove_many(sp_std::iter::once(id)); + Ok(()) + } + + /// Remove many ids from the list. + /// + /// This is more efficient than repeated calls to `Self::remove`. + /// + /// Returns the final count of number of ids removed. + fn remove_many<'a>(ids: impl IntoIterator) -> u32 { + let mut bags = BTreeMap::new(); + let mut count = 0; + + for id in ids.into_iter() { + let node = match Node::::get(id) { + Some(node) => node, + None => continue, + }; + count += 1; + + if !node.is_terminal() { + // this node is not a head or a tail and thus the bag does not need to be updated + node.excise() + } else { + // this node is a head or tail, so the bag needs to be updated + let bag = bags + .entry(node.bag_upper) + .or_insert_with(|| Bag::::get_or_make(node.bag_upper)); + // node.bag_upper must be correct, therefore this bag will contain this node. + bag.remove_node_unchecked(&node); + } + + // now get rid of the node itself + node.remove_from_storage_unchecked() + } + + for (_, bag) in bags { + bag.put(); + } + + count + } + + /// Update a node's position in the list. + /// + /// If the node was in the correct bag, no effect. If the node was in the incorrect bag, they + /// are moved into the correct bag. + /// + /// Returns `Some((old_idx, new_idx))` if the node moved, otherwise `None`. In both cases, the + /// node's score is written to the `score` field. Thus, this is not a noop, even if `None`. + /// + /// This operation is somewhat more efficient than simply calling [`self.remove`] followed by + /// [`self.insert`]. However, given large quantities of nodes to move, it may be more efficient + /// to call [`self.remove_many`] followed by [`self.insert_many`]. + pub(crate) fn update_position_for( + mut node: Node, + new_score: T::Score, + ) -> Option<(T::Score, T::Score)> { + node.score = new_score; + if node.is_misplaced(new_score) { + let old_bag_upper = node.bag_upper; + + if !node.is_terminal() { + // this node is not a head or a tail, so we can just cut it out of the list. update + // and put the prev and next of this node, we do `node.put` inside `insert_note`. + node.excise(); + } else if let Some(mut bag) = Bag::::get(node.bag_upper) { + // this is a head or tail, so the bag must be updated. + bag.remove_node_unchecked(&node); + bag.put(); + } else { + frame_support::defensive!( + "Node did not have a bag; BagsList is in an inconsistent state" + ); + } + + // put the node into the appropriate new bag. + let new_bag_upper = notional_bag_for::(new_score); + let mut bag = Bag::::get_or_make(new_bag_upper); + // prev, next, and bag_upper of the node are updated inside `insert_node`, also + // `node.put` is in there. + bag.insert_node_unchecked(node); + bag.put(); + + Some((old_bag_upper, new_bag_upper)) + } else { + // just write the new score. + node.put(); + None + } + } + + /// Put `heavier_id` to the position directly in front of `lighter_id`. Both ids must be in the + /// same bag and the `score_of` `lighter_id` must be less than that of `heavier_id`. + pub(crate) fn put_in_front_of( + lighter_id: &T::AccountId, + heavier_id: &T::AccountId, + ) -> Result<(), ListError> { + let lighter_node = Node::::get(&lighter_id).ok_or(ListError::NodeNotFound)?; + let heavier_node = Node::::get(&heavier_id).ok_or(ListError::NodeNotFound)?; + + ensure!(lighter_node.bag_upper == heavier_node.bag_upper, ListError::NotInSameBag); + + // this is the most expensive check, so we do it last. + ensure!( + T::ScoreProvider::score(&heavier_id) > T::ScoreProvider::score(&lighter_id), + ListError::NotHeavier + ); + + // remove the heavier node from this list. Note that this removes the node from storage and + // decrements the node counter. + let _ = + Self::remove(&heavier_id).defensive_proof("both nodes have been checked to exist; qed"); + + // re-fetch `lighter_node` from storage since it may have been updated when `heavier_node` + // was removed. + let lighter_node = + Node::::get(lighter_id).defensive_ok_or_else(|| ListError::NodeNotFound)?; + + // insert `heavier_node` directly in front of `lighter_node`. This will update both nodes + // in storage and update the node counter. + Self::insert_at_unchecked(lighter_node, heavier_node); + + Ok(()) + } + + /// Insert `node` directly in front of `at`. + /// + /// WARNINGS: + /// - this is a naive function in that it does not check if `node` belongs to the same bag as + /// `at`. It is expected that the call site will check preconditions. + /// - this will panic if `at.bag_upper` is not a bag that already exists in storage. + fn insert_at_unchecked(mut at: Node, mut node: Node) { + // connect `node` to its new `prev`. + node.prev = at.prev.clone(); + if let Some(mut prev) = at.prev() { + prev.next = Some(node.id().clone()); + prev.put() + } + + // connect `node` and `at`. + node.next = Some(at.id().clone()); + at.prev = Some(node.id().clone()); + + if node.is_terminal() { + // `node` is the new head, so we make sure the bag is updated. Note, + // since `node` is always in front of `at` we know that 1) there is always at least 2 + // nodes in the bag, and 2) only `node` could be the head and only `at` could be the + // tail. + let mut bag = Bag::::get(at.bag_upper) + .expect("given nodes must always have a valid bag. qed."); + + if node.prev == None { + bag.head = Some(node.id().clone()) + } + + bag.put() + }; + + // write the updated nodes to storage. + at.put(); + node.put(); + } + + /// Check the internal state of the list. + /// + /// This should be called from the call-site, whenever one of the mutating apis (e.g. `insert`) + /// is being used, after all other staking data (such as counter) has been updated. It checks: + /// + /// * there are no duplicate ids, + /// * length of this list is in sync with `ListNodes::count()`, + /// * and sanity-checks all bags and nodes. This will cascade down all the checks and makes sure + /// all bags and nodes are checked per *any* update to `List`. + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + pub(crate) fn do_try_state() -> Result<(), TryRuntimeError> { + let mut seen_in_list = BTreeSet::new(); + ensure!( + Self::iter().map(|node| node.id).all(|id| seen_in_list.insert(id)), + "duplicate identified" + ); + + let iter_count = Self::iter().count() as u32; + let stored_count = crate::ListNodes::::count(); + let nodes_count = crate::ListNodes::::iter().count() as u32; + ensure!(iter_count == stored_count, "iter_count != stored_count"); + ensure!(stored_count == nodes_count, "stored_count != nodes_count"); + + crate::log!(trace, "count of nodes: {}", stored_count); + + let active_bags = { + let thresholds = T::BagThresholds::get().iter().copied(); + let thresholds: Vec = + if thresholds.clone().last() == Some(T::Score::max_value()) { + // in the event that they included it, we don't need to make any changes + thresholds.collect() + } else { + // otherwise, insert it here. + thresholds.chain(iter::once(T::Score::max_value())).collect() + }; + thresholds.into_iter().filter_map(|t| Bag::::get(t)) + }; + + let _ = active_bags.clone().try_for_each(|b| b.do_try_state())?; + + let nodes_in_bags_count = + active_bags.clone().fold(0u32, |acc, cur| acc + cur.iter().count() as u32); + ensure!(nodes_count == nodes_in_bags_count, "stored_count != nodes_in_bags_count"); + + crate::log!(trace, "count of active bags {}", active_bags.count()); + + // check that all nodes are sane. We check the `ListNodes` storage item directly in case we + // have some "stale" nodes that are not in a bag. + for (_id, node) in crate::ListNodes::::iter() { + node.do_try_state()? + } + + Ok(()) + } + + /// Returns the nodes of all non-empty bags. For testing and benchmarks. + #[cfg(any(feature = "std", feature = "runtime-benchmarks"))] + #[allow(dead_code)] + pub(crate) fn get_bags() -> Vec<(T::Score, Vec)> { + use frame_support::traits::Get as _; + + let thresholds = T::BagThresholds::get(); + let iter = thresholds.iter().copied(); + let iter: Box> = if thresholds.last() == + Some(&T::Score::max_value()) + { + // in the event that they included it, we can just pass the iterator through unchanged. + Box::new(iter) + } else { + // otherwise, insert it here. + Box::new(iter.chain(sp_std::iter::once(T::Score::max_value()))) + }; + + iter.filter_map(|t| { + Bag::::get(t) + .map(|bag| (t, bag.iter().map(|n| n.id().clone()).collect::>())) + }) + .collect::>() + } +} + +/// A Bag is a doubly-linked list of ids, where each id is mapped to a [`Node`]. +/// +/// Note that we maintain both head and tail pointers. While it would be possible to get away with +/// maintaining only a head pointer and cons-ing elements onto the front of the list, it's more +/// desirable to ensure that there is some element of first-come, first-serve to the list's +/// iteration so that there's no incentive to churn ids positioning to improve the chances of +/// appearing within the ids set. +#[derive(DefaultNoBound, Encode, Decode, MaxEncodedLen, TypeInfo)] +#[codec(mel_bound())] +#[scale_info(skip_type_params(T, I))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] +pub struct Bag, I: 'static = ()> { + head: Option, + tail: Option, + + #[codec(skip)] + bag_upper: T::Score, + #[codec(skip)] + _phantom: PhantomData, +} + +impl, I: 'static> Bag { + #[cfg(test)] + pub(crate) fn new( + head: Option, + tail: Option, + bag_upper: T::Score, + ) -> Self { + Self { head, tail, bag_upper, _phantom: PhantomData } + } + + /// Get a bag by its upper score. + pub(crate) fn get(bag_upper: T::Score) -> Option> { + crate::ListBags::::try_get(bag_upper).ok().map(|mut bag| { + bag.bag_upper = bag_upper; + bag + }) + } + + /// Get a bag by its upper score or make it, appropriately initialized. Does not check if + /// if `bag_upper` is a valid threshold. + fn get_or_make(bag_upper: T::Score) -> Bag { + Self::get(bag_upper).unwrap_or(Bag { bag_upper, ..Default::default() }) + } + + /// `True` if self is empty. + fn is_empty(&self) -> bool { + self.head.is_none() && self.tail.is_none() + } + + /// Put the bag back into storage. + fn put(self) { + if self.is_empty() { + crate::ListBags::::remove(self.bag_upper); + } else { + crate::ListBags::::insert(self.bag_upper, self); + } + } + + /// Get the head node in this bag. + fn head(&self) -> Option> { + self.head.as_ref().and_then(|id| Node::get(id)) + } + + /// Get the tail node in this bag. + fn tail(&self) -> Option> { + self.tail.as_ref().and_then(|id| Node::get(id)) + } + + /// Iterate over the nodes in this bag. + pub(crate) fn iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Insert a new id into this bag. + /// + /// This is private on purpose because it's naive: it doesn't check whether this is the + /// appropriate bag for this id at all. Generally, use [`List::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the nodes. You still need to call + /// `self.put()` after use. + fn insert_unchecked(&mut self, id: T::AccountId, score: T::Score) { + // insert_node will overwrite `prev`, `next` and `bag_upper` to the proper values. As long + // as this bag is the correct one, we're good. All calls to this must come after getting the + // correct [`notional_bag_for`]. + self.insert_node_unchecked(Node:: { + id, + prev: None, + next: None, + bag_upper: Zero::zero(), + score, + _phantom: PhantomData, + }); + } + + /// Insert a node into this bag. + /// + /// This is private on purpose because it's naive; it doesn't check whether this is the + /// appropriate bag for this node at all. Generally, use [`List::insert`] instead. + /// + /// Storage note: this modifies storage, but only for the node. You still need to call + /// `self.put()` after use. + fn insert_node_unchecked(&mut self, mut node: Node) { + if let Some(tail) = &self.tail { + if *tail == node.id { + // this should never happen, but this check prevents one path to a worst case + // infinite loop. + defensive!("system logic error: inserting a node who has the id of tail"); + return + }; + } + + // re-set the `bag_upper`. Regardless of whatever the node had previously, now it is going + // to be `self.bag_upper`. + node.bag_upper = self.bag_upper; + + let id = node.id.clone(); + // update this node now, treating it as the new tail. + node.prev = self.tail.clone(); + node.next = None; + node.put(); + + // update the previous tail. + if let Some(mut old_tail) = self.tail() { + old_tail.next = Some(id.clone()); + old_tail.put(); + } + self.tail = Some(id.clone()); + + // ensure head exist. This is only set when the length of the bag is just 1, i.e. if this is + // the first insertion into the bag. In this case, both head and tail should point to the + // same node. + if self.head.is_none() { + self.head = Some(id); + debug_assert!(self.iter().count() == 1); + } + } + + /// Remove a node from this bag. + /// + /// This is private on purpose because it doesn't check whether this bag contains the node in + /// the first place. Generally, use [`List::remove`] instead, similar to `insert_unchecked`. + /// + /// Storage note: this modifies storage, but only for adjacent nodes. You still need to call + /// `self.put()` and `ListNodes::remove(id)` to update storage for the bag and `node`. + fn remove_node_unchecked(&mut self, node: &Node) { + // reassign neighboring nodes. + node.excise(); + + // clear the bag head/tail pointers as necessary. + if self.tail.as_ref() == Some(&node.id) { + self.tail = node.prev.clone(); + } + if self.head.as_ref() == Some(&node.id) { + self.head = node.next.clone(); + } + } + + /// Check the internal state of the bag. + /// + /// Should be called by the call-site, after any mutating operation on a bag. The call site of + /// this struct is always `List`. + /// + /// * Ensures head has no prev. + /// * Ensures tail has no next. + /// * Ensures there are no loops, traversal from head to tail is correct. + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + fn do_try_state(&self) -> Result<(), TryRuntimeError> { + frame_support::ensure!( + self.head() + .map(|head| head.prev().is_none()) + // if there is no head, then there must not be a tail, meaning that the bag is + // empty. + .unwrap_or_else(|| self.tail.is_none()), + "head has a prev" + ); + + frame_support::ensure!( + self.tail() + .map(|tail| tail.next().is_none()) + // if there is no tail, then there must not be a head, meaning that the bag is + // empty. + .unwrap_or_else(|| self.head.is_none()), + "tail has a next" + ); + + let mut seen_in_bag = BTreeSet::new(); + frame_support::ensure!( + self.iter() + .map(|node| node.id) + // each voter is only seen once, thus there is no cycle within a bag + .all(|voter| seen_in_bag.insert(voter)), + "duplicate found in bag" + ); + + Ok(()) + } + + /// Iterate over the nodes in this bag (public for tests). + #[cfg(feature = "std")] + #[allow(dead_code)] + pub fn std_iter(&self) -> impl Iterator> { + sp_std::iter::successors(self.head(), |prev| prev.next()) + } + + /// Check if the bag contains a node with `id`. + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + fn contains(&self, id: &T::AccountId) -> bool { + self.iter().any(|n| n.id() == id) + } +} + +/// A Node is the fundamental element comprising the doubly-linked list described by `Bag`. +#[derive(Encode, Decode, MaxEncodedLen, TypeInfo)] +#[codec(mel_bound())] +#[scale_info(skip_type_params(T, I))] +#[cfg_attr(feature = "std", derive(frame_support::DebugNoBound, Clone, PartialEq))] +pub struct Node, I: 'static = ()> { + pub(crate) id: T::AccountId, + pub(crate) prev: Option, + pub(crate) next: Option, + pub(crate) bag_upper: T::Score, + pub(crate) score: T::Score, + #[codec(skip)] + pub(crate) _phantom: PhantomData, +} + +impl, I: 'static> Node { + /// Get a node by id. + pub fn get(id: &T::AccountId) -> Option> { + crate::ListNodes::::try_get(id).ok() + } + + /// Put the node back into storage. + fn put(self) { + crate::ListNodes::::insert(self.id.clone(), self); + } + + /// Update neighboring nodes to point to reach other. + /// + /// Only updates storage for adjacent nodes, but not `self`; so the user may need to call + /// `self.put`. + fn excise(&self) { + // Update previous node. + if let Some(mut prev) = self.prev() { + prev.next = self.next.clone(); + prev.put(); + } + // Update next self. + if let Some(mut next) = self.next() { + next.prev = self.prev.clone(); + next.put(); + } + } + + /// This is a naive function that removes a node from the `ListNodes` storage item. + /// + /// It is naive because it does not check if the node has first been removed from its bag. + fn remove_from_storage_unchecked(&self) { + crate::ListNodes::::remove(&self.id) + } + + /// Get the previous node in the bag. + fn prev(&self) -> Option> { + self.prev.as_ref().and_then(|id| Node::get(id)) + } + + /// Get the next node in the bag. + fn next(&self) -> Option> { + self.next.as_ref().and_then(|id| Node::get(id)) + } + + /// `true` when this voter is in the wrong bag. + pub fn is_misplaced(&self, current_score: T::Score) -> bool { + notional_bag_for::(current_score) != self.bag_upper + } + + /// `true` when this voter is a bag head or tail. + fn is_terminal(&self) -> bool { + self.prev.is_none() || self.next.is_none() + } + + /// Get the underlying voter. + pub(crate) fn id(&self) -> &T::AccountId { + &self.id + } + + /// Get the current vote weight of the node. + pub(crate) fn score(&self) -> T::Score { + self.score + } + + /// Get the underlying voter (public fo tests). + #[cfg(feature = "std")] + #[allow(dead_code)] + pub fn std_id(&self) -> &T::AccountId { + &self.id + } + + #[cfg(any(feature = "runtime-benchmarks", feature = "fuzz", test))] + pub fn set_score(&mut self, s: T::Score) { + self.score = s + } + + /// The bag this nodes belongs to (public for benchmarks). + #[cfg(feature = "runtime-benchmarks")] + #[allow(dead_code)] + pub fn bag_upper(&self) -> T::Score { + self.bag_upper + } + + #[cfg(any(test, feature = "try-runtime", feature = "fuzz"))] + fn do_try_state(&self) -> Result<(), TryRuntimeError> { + let expected_bag = Bag::::get(self.bag_upper).ok_or("bag not found for node")?; + + let id = self.id(); + + frame_support::ensure!(expected_bag.contains(id), "node does not exist in the bag"); + + let non_terminal_check = !self.is_terminal() && + expected_bag.head.as_ref() != Some(id) && + expected_bag.tail.as_ref() != Some(id); + let terminal_check = + expected_bag.head.as_ref() == Some(id) || expected_bag.tail.as_ref() == Some(id); + frame_support::ensure!( + non_terminal_check || terminal_check, + "a terminal node is neither its bag head or tail" + ); + + Ok(()) + } +} diff --git a/frame/bags-list/src/tests_bags_list.rs b/frame/balances/bags-list/src/list/tests.rs similarity index 91% rename from frame/bags-list/src/tests_bags_list.rs rename to frame/balances/bags-list/src/list/tests.rs index 2b16e94f2451c..fd4ad8f893af3 100644 --- a/frame/bags-list/src/tests_bags_list.rs +++ b/frame/balances/bags-list/src/list/tests.rs @@ -15,18 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::*; use crate::{ - Config, mock::{test_utils::*, *}, - List, Bag, Node, ListError, - notional_bag_for, + ListBags, ListNodes, }; -use core::marker::PhantomData; -use frame_election_provider_support::{SortedListProvider, ScoreProvider, VoteWeight}; +use frame_election_provider_support::{SortedListProvider, VoteWeight}; use frame_support::{assert_ok, assert_storage_noop}; -use std::iter; - -//TODO: These tests should move under `bags-list` crate +use sp_runtime::TryRuntimeError; fn node( id: AccountId, @@ -40,29 +36,29 @@ fn node( #[test] fn basic_setup_works() { ExtBuilder::default().build_and_execute(|| { - assert_eq!(List::::count(), 4); - assert_eq!(List::::iter().count(), 4); - assert_eq!(List::::bag_count(), 2); + assert_eq!(ListNodes::::count(), 4); + assert_eq!(ListNodes::::iter().count(), 4); + assert_eq!(ListBags::::iter().count(), 2); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); // the state of the bags is as expected assert_eq!( - Bag::::get_raw(10).unwrap(), + ListBags::::get(10).unwrap(), Bag:: { head: Some(1), tail: Some(1), bag_upper: 0, _phantom: PhantomData } ); assert_eq!( - Bag::::get_raw(1_000).unwrap(), + ListBags::::get(1_000).unwrap(), Bag:: { head: Some(2), tail: Some(4), bag_upper: 0, _phantom: PhantomData } ); - assert_eq!(Node::::get(&2).unwrap(), node(2, None, Some(3), 1_000)); - assert_eq!(Node::::get(&3).unwrap(), node(3, Some(2), Some(4), 1_000)); - assert_eq!(Node::::get(&4).unwrap(), node(4, Some(3), None, 1_000)); - assert_eq!(Node::::get(&1).unwrap(), node(1, None, None, 10)); + assert_eq!(ListNodes::::get(2).unwrap(), node(2, None, Some(3), 1_000)); + assert_eq!(ListNodes::::get(3).unwrap(), node(3, Some(2), Some(4), 1_000)); + assert_eq!(ListNodes::::get(4).unwrap(), node(4, Some(3), None, 1_000)); + assert_eq!(ListNodes::::get(1).unwrap(), node(1, None, None, 10)); // non-existent id does not have a storage footprint - assert_eq!(Node::::get(&42), None); + assert_eq!(ListNodes::::get(42), None); // iteration of the bags would yield: assert_eq!( @@ -161,6 +157,7 @@ fn migrate_works() { mod list { use frame_support::assert_noop; + use super::*; #[test] @@ -258,16 +255,17 @@ mod list { } #[test] - fn remove_works() { + fn remove_works() { + use crate::{ListBags, ListNodes}; let ensure_left = |id, counter| { - assert!(!List::::contains(id)); - assert_eq!(List::::count(), counter); - assert_eq!(List::::iter().count() as u32, counter); + assert!(!ListNodes::::contains_key(id)); + assert_eq!(ListNodes::::count(), counter); + assert_eq!(ListNodes::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { // removing a non-existent id is a noop - assert!(!List::::contains(&42)); + assert!(!ListNodes::::contains_key(42)); assert_noop!(List::::remove(&42), ListError::NodeNotFound); // when removing a node from a bag with multiple nodes: @@ -276,7 +274,7 @@ mod list { // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); - ensure_left(&2, 3); + ensure_left(2, 3); // when removing a node from a bag with only one node: List::::remove(&1).unwrap(); @@ -284,21 +282,21 @@ mod list { // then assert_eq!(get_list_as_ids(), vec![3, 4]); assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); - ensure_left(&1, 2); + ensure_left(1, 2); // bag 10 is removed - assert_eq!(Bag::::get_raw(10), None); - + assert!(!ListBags::::contains_key(10)); + // remove remaining ids to make sure storage cleans up as expected List::::remove(&3).unwrap(); - ensure_left(&3, 1); + ensure_left(3, 1); assert_eq!(get_list_as_ids(), vec![4]); List::::remove(&4).unwrap(); - ensure_left(&4, 0); + ensure_left(4, 0); assert_eq!(get_list_as_ids(), Vec::::new()); // bags are deleted via removals - assert_eq!(List::::bag_count(), 0); + assert_eq!(ListBags::::iter().count(), 0); }); } @@ -362,21 +360,27 @@ mod list { // make sure there are no duplicates. ExtBuilder::default().build_and_execute_no_post_check(|| { Bag::::get(10).unwrap().insert_unchecked(2, 10); - assert_eq!(List::::do_try_state(), Err("duplicate identified")); + assert_eq!( + List::::do_try_state(), + TryRuntimeError::Other("duplicate identified").into() + ); }); // ensure count is in sync with `ListNodes::count()`. ExtBuilder::default().build_and_execute_no_post_check(|| { - assert_eq!(List::::count(), 4); + assert_eq!(crate::ListNodes::::count(), 4); // we do some wacky stuff here to get access to the counter, since it is (reasonably) // not exposed as mutable in any sense. #[frame_support::storage_alias] type CounterForListNodes = StorageValue, u32, frame_support::pallet_prelude::ValueQuery>; CounterForListNodes::::mutate(|counter| *counter += 1); - assert_eq!(List::::count(), 5); + assert_eq!(crate::ListNodes::::count(), 5); - assert_eq!(List::::do_try_state(), Err("iter_count != stored_count")); + assert_eq!( + List::::do_try_state(), + TryRuntimeError::Other("iter_count != stored_count").into() + ); }); } @@ -412,11 +416,11 @@ mod list { }; // given - List::::insert_unchecked(10, node_10_no_bag); - List::::insert_unchecked(11, node_11_no_bag); + ListNodes::::insert(10, node_10_no_bag); + ListNodes::::insert(11, node_11_no_bag); StakingMock::set_score_of(&10, 14); StakingMock::set_score_of(&11, 15); - assert_eq!(Bag::::get(15), None); + assert!(!ListBags::::contains_key(15)); assert_eq!(List::::get_bags(), vec![]); // then .. this panics @@ -442,9 +446,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!List::::contains(&42)); + assert!(!crate::ListNodes::::contains_key(42)); - let node_1 = Node::::get(&1).unwrap(); + let node_1 = crate::ListNodes::::get(&1).unwrap(); // when List::::insert_at_unchecked(node_1, node_42); @@ -472,9 +476,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!List::::contains(&42)); + assert!(!crate::ListNodes::::contains_key(42)); - let node_2 = Node::::get(&2).unwrap(); + let node_2 = crate::ListNodes::::get(&2).unwrap(); // when List::::insert_at_unchecked(node_2, node_42); @@ -502,9 +506,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!List::::contains(&42)); + assert!(!crate::ListNodes::::contains_key(42)); - let node_3 = Node::::get(&3).unwrap(); + let node_3 = crate::ListNodes::::get(&3).unwrap(); // when List::::insert_at_unchecked(node_3, node_42); @@ -532,9 +536,9 @@ mod list { score: 1_000, _phantom: PhantomData, }; - assert!(!List::::contains(&42)); + assert!(!crate::ListNodes::::contains_key(42)); - let node_4 = Node::::get(&4).unwrap(); + let node_4 = crate::ListNodes::::get(&4).unwrap(); // when List::::insert_at_unchecked(node_4, node_42); @@ -575,7 +579,7 @@ mod bags { .filter(|bag_upper| !vec![10, 1_000].contains(bag_upper)) .for_each(|bag_upper| { assert_storage_noop!(assert_eq!(Bag::::get(*bag_upper), None)); - assert_eq!(Bag::::get(*bag_upper), None); + assert!(!ListBags::::contains_key(*bag_upper)); }); // when we make a pre-existing bag empty @@ -604,7 +608,7 @@ mod bags { bag_10.insert_node_unchecked(node(42, 5)); assert_eq!( - Node::::get(&42).unwrap(), + ListNodes::::get(&42).unwrap(), Node { bag_upper: 10, score: 5, diff --git a/frame/bags-list/src/migrations.rs b/frame/balances/bags-list/src/migrations.rs similarity index 92% rename from frame/bags-list/src/migrations.rs rename to frame/balances/bags-list/src/migrations.rs index e5c8cad0c58ec..7df63a6a44c54 100644 --- a/frame/bags-list/src/migrations.rs +++ b/frame/balances/bags-list/src/migrations.rs @@ -24,6 +24,9 @@ use frame_support::traits::OnRuntimeUpgrade; #[cfg(feature = "try-runtime")] use frame_support::ensure; +#[cfg(feature = "try-runtime")] +use sp_runtime::TryRuntimeError; + #[cfg(feature = "try-runtime")] use sp_std::vec::Vec; @@ -35,7 +38,7 @@ impl, I: 'static> OnRuntimeUpgrade for CheckCounterPrefix Result, &'static str> { + fn pre_upgrade() -> Result, TryRuntimeError> { // The old explicit storage item. #[frame_support::storage_alias] type CounterForListNodes, I: 'static> = @@ -88,7 +91,7 @@ mod old { pub struct AddScore, I: 'static = ()>(sp_std::marker::PhantomData<(T, I)>); impl, I: 'static> OnRuntimeUpgrade for AddScore { #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result, &'static str> { + fn pre_upgrade() -> Result, TryRuntimeError> { // The list node data should be corrupt at this point, so this is zero. ensure!(crate::ListNodes::::iter().count() == 0, "list node data is not corrupt"); // We can use the helper `old::ListNode` to get the existing data. @@ -103,7 +106,7 @@ impl, I: 'static> OnRuntimeUpgrade for AddScore { for (_key, node) in old::ListNodes::::iter() { let score = T::ScoreProvider::score(&node.id); - let new_node = crate::Node:: { + let new_node = crate::Node { id: node.id.clone(), prev: node.prev, next: node.next, @@ -112,14 +115,14 @@ impl, I: 'static> OnRuntimeUpgrade for AddScore { _phantom: node._phantom, }; - new_node.put(); + crate::ListNodes::::insert(node.id, new_node); } return frame_support::weights::Weight::MAX } #[cfg(feature = "try-runtime")] - fn post_upgrade(node_count_before: Vec) -> Result<(), &'static str> { + fn post_upgrade(node_count_before: Vec) -> Result<(), TryRuntimeError> { let node_count_before: u32 = Decode::decode(&mut node_count_before.as_slice()) .expect("the state parameter should be something that was generated by pre_upgrade"); // Now the list node data is not corrupt anymore. diff --git a/frame/bags-list/src/mock.rs b/frame/balances/bags-list/src/mock.rs similarity index 95% rename from frame/bags-list/src/mock.rs rename to frame/balances/bags-list/src/mock.rs index 1f94b6525a92f..efbb2ed94c49f 100644 --- a/frame/bags-list/src/mock.rs +++ b/frame/balances/bags-list/src/mock.rs @@ -40,9 +40,10 @@ impl frame_election_provider_support::ScoreProvider for StakingMock { *NextVoteWeightMap::get().get(id).unwrap_or(&NextVoteWeight::get()) } - #[cfg(any(feature = "runtime-benchmarks", feature = "fuzz", test))] - fn set_score_of(id: &AccountId, weight: Self::Score) { - NEXT_VOTE_WEIGHT_MAP.with(|m| m.borrow_mut().insert(*id, weight)); + frame_election_provider_support::runtime_benchmarks_fuzz_or_std_enabled! { + fn set_score_of(id: &AccountId, weight: Self::Score) { + NEXT_VOTE_WEIGHT_MAP.with(|m| m.borrow_mut().insert(*id, weight)); + } } } @@ -161,7 +162,7 @@ impl ExtBuilder { #[cfg(test)] pub(crate) mod test_utils { use super::*; - use crate::Bag; + use list::Bag; /// Returns the ordered ids within the given bag. pub(crate) fn bag_as_ids(bag: &Bag) -> Vec { diff --git a/frame/bags-list/src/tests.rs b/frame/balances/bags-list/src/tests.rs similarity index 97% rename from frame/bags-list/src/tests.rs rename to frame/balances/bags-list/src/tests.rs index 288c624879ebd..74f1491835a32 100644 --- a/frame/bags-list/src/tests.rs +++ b/frame/balances/bags-list/src/tests.rs @@ -19,7 +19,7 @@ use frame_support::{assert_noop, assert_ok, assert_storage_noop, traits::Integri use super::*; use frame_election_provider_support::{SortedListProvider, VoteWeight}; -use crate::{Bag, Node}; +use list::Bag; use mock::{test_utils::*, *}; mod pallet { @@ -153,7 +153,7 @@ mod pallet { #[test] fn wrong_rebag_errs() { ExtBuilder::default().build_and_execute(|| { - let node_3 = Node::::get(&3).unwrap(); + let node_3 = list::Node::::get(&3).unwrap(); // when account 3 is _not_ misplaced with score 500 NextVoteWeight::set(500); assert!(!node_3.is_misplaced(500)); @@ -412,7 +412,7 @@ mod pallet { // given assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); - assert!(!List::::contains(&5)); + assert!(!ListNodes::::contains_key(5)); // then assert_noop!( @@ -426,7 +426,7 @@ mod pallet { // given assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![2, 3, 4])]); - assert!(!List::::contains(&5)); + assert!(!ListNodes::::contains_key(5)); // then assert_noop!( @@ -612,15 +612,15 @@ mod sorted_list_provider { #[test] fn on_remove_works() { let ensure_left = |id, counter| { - assert!(!List::::contains(id)); + assert!(!ListNodes::::contains_key(id)); assert_eq!(BagsList::count(), counter); - assert_eq!(List::::count(), counter); - assert_eq!(List::::iter().count() as u32, counter); + assert_eq!(ListNodes::::count(), counter); + assert_eq!(ListNodes::::iter().count() as u32, counter); }; ExtBuilder::default().build_and_execute(|| { // it is a noop removing a non-existent id - assert!(!List::::contains(&42)); + assert!(!ListNodes::::contains_key(42)); assert_noop!(BagsList::on_remove(&42), ListError::NodeNotFound); // when removing a node from a bag with multiple nodes @@ -629,7 +629,7 @@ mod sorted_list_provider { // then assert_eq!(get_list_as_ids(), vec![3, 4, 1]); assert_eq!(List::::get_bags(), vec![(10, vec![1]), (1_000, vec![3, 4])]); - ensure_left(&2, 3); + ensure_left(2, 3); // when removing a node from a bag with only one node BagsList::on_remove(&1).unwrap(); @@ -637,17 +637,17 @@ mod sorted_list_provider { // then assert_eq!(get_list_as_ids(), vec![3, 4]); assert_eq!(List::::get_bags(), vec![(1_000, vec![3, 4])]); - ensure_left(&1, 2); + ensure_left(1, 2); // when removing all remaining ids BagsList::on_remove(&4).unwrap(); assert_eq!(get_list_as_ids(), vec![3]); - ensure_left(&4, 1); + ensure_left(4, 1); BagsList::on_remove(&3).unwrap(); // then the storage is completely cleaned up assert_eq!(get_list_as_ids(), Vec::::new()); - ensure_left(&3, 0); + ensure_left(3, 0); }); } diff --git a/frame/bags-list/src/weights.rs b/frame/balances/bags-list/src/weights.rs similarity index 100% rename from frame/bags-list/src/weights.rs rename to frame/balances/bags-list/src/weights.rs From 03f639c36f26ac355a485868085e9b765bc88a2e Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Thu, 25 May 2023 12:06:51 +0530 Subject: [PATCH 18/55] Fix structure --- frame/{balances => }/bags-list/Cargo.toml | 0 frame/{balances => }/bags-list/fuzzer/.gitignore | 0 frame/{balances => }/bags-list/fuzzer/Cargo.toml | 0 frame/{balances => }/bags-list/fuzzer/src/main.rs | 0 frame/{balances => }/bags-list/remote-tests/Cargo.toml | 0 frame/{balances => }/bags-list/remote-tests/src/lib.rs | 0 frame/{balances => }/bags-list/remote-tests/src/migration.rs | 0 frame/{balances => }/bags-list/remote-tests/src/snapshot.rs | 0 frame/{balances => }/bags-list/remote-tests/src/try_state.rs | 0 frame/{balances => }/bags-list/src/benchmarks.rs | 0 frame/{balances => }/bags-list/src/lib.rs | 0 frame/{balances => }/bags-list/src/list/mod.rs | 0 frame/{balances => }/bags-list/src/list/tests.rs | 0 frame/{balances => }/bags-list/src/migrations.rs | 0 frame/{balances => }/bags-list/src/mock.rs | 0 frame/{balances => }/bags-list/src/tests.rs | 0 frame/{balances => }/bags-list/src/weights.rs | 0 17 files changed, 0 insertions(+), 0 deletions(-) rename frame/{balances => }/bags-list/Cargo.toml (100%) rename frame/{balances => }/bags-list/fuzzer/.gitignore (100%) rename frame/{balances => }/bags-list/fuzzer/Cargo.toml (100%) rename frame/{balances => }/bags-list/fuzzer/src/main.rs (100%) rename frame/{balances => }/bags-list/remote-tests/Cargo.toml (100%) rename frame/{balances => }/bags-list/remote-tests/src/lib.rs (100%) rename frame/{balances => }/bags-list/remote-tests/src/migration.rs (100%) rename frame/{balances => }/bags-list/remote-tests/src/snapshot.rs (100%) rename frame/{balances => }/bags-list/remote-tests/src/try_state.rs (100%) rename frame/{balances => }/bags-list/src/benchmarks.rs (100%) rename frame/{balances => }/bags-list/src/lib.rs (100%) rename frame/{balances => }/bags-list/src/list/mod.rs (100%) rename frame/{balances => }/bags-list/src/list/tests.rs (100%) rename frame/{balances => }/bags-list/src/migrations.rs (100%) rename frame/{balances => }/bags-list/src/mock.rs (100%) rename frame/{balances => }/bags-list/src/tests.rs (100%) rename frame/{balances => }/bags-list/src/weights.rs (100%) diff --git a/frame/balances/bags-list/Cargo.toml b/frame/bags-list/Cargo.toml similarity index 100% rename from frame/balances/bags-list/Cargo.toml rename to frame/bags-list/Cargo.toml diff --git a/frame/balances/bags-list/fuzzer/.gitignore b/frame/bags-list/fuzzer/.gitignore similarity index 100% rename from frame/balances/bags-list/fuzzer/.gitignore rename to frame/bags-list/fuzzer/.gitignore diff --git a/frame/balances/bags-list/fuzzer/Cargo.toml b/frame/bags-list/fuzzer/Cargo.toml similarity index 100% rename from frame/balances/bags-list/fuzzer/Cargo.toml rename to frame/bags-list/fuzzer/Cargo.toml diff --git a/frame/balances/bags-list/fuzzer/src/main.rs b/frame/bags-list/fuzzer/src/main.rs similarity index 100% rename from frame/balances/bags-list/fuzzer/src/main.rs rename to frame/bags-list/fuzzer/src/main.rs diff --git a/frame/balances/bags-list/remote-tests/Cargo.toml b/frame/bags-list/remote-tests/Cargo.toml similarity index 100% rename from frame/balances/bags-list/remote-tests/Cargo.toml rename to frame/bags-list/remote-tests/Cargo.toml diff --git a/frame/balances/bags-list/remote-tests/src/lib.rs b/frame/bags-list/remote-tests/src/lib.rs similarity index 100% rename from frame/balances/bags-list/remote-tests/src/lib.rs rename to frame/bags-list/remote-tests/src/lib.rs diff --git a/frame/balances/bags-list/remote-tests/src/migration.rs b/frame/bags-list/remote-tests/src/migration.rs similarity index 100% rename from frame/balances/bags-list/remote-tests/src/migration.rs rename to frame/bags-list/remote-tests/src/migration.rs diff --git a/frame/balances/bags-list/remote-tests/src/snapshot.rs b/frame/bags-list/remote-tests/src/snapshot.rs similarity index 100% rename from frame/balances/bags-list/remote-tests/src/snapshot.rs rename to frame/bags-list/remote-tests/src/snapshot.rs diff --git a/frame/balances/bags-list/remote-tests/src/try_state.rs b/frame/bags-list/remote-tests/src/try_state.rs similarity index 100% rename from frame/balances/bags-list/remote-tests/src/try_state.rs rename to frame/bags-list/remote-tests/src/try_state.rs diff --git a/frame/balances/bags-list/src/benchmarks.rs b/frame/bags-list/src/benchmarks.rs similarity index 100% rename from frame/balances/bags-list/src/benchmarks.rs rename to frame/bags-list/src/benchmarks.rs diff --git a/frame/balances/bags-list/src/lib.rs b/frame/bags-list/src/lib.rs similarity index 100% rename from frame/balances/bags-list/src/lib.rs rename to frame/bags-list/src/lib.rs diff --git a/frame/balances/bags-list/src/list/mod.rs b/frame/bags-list/src/list/mod.rs similarity index 100% rename from frame/balances/bags-list/src/list/mod.rs rename to frame/bags-list/src/list/mod.rs diff --git a/frame/balances/bags-list/src/list/tests.rs b/frame/bags-list/src/list/tests.rs similarity index 100% rename from frame/balances/bags-list/src/list/tests.rs rename to frame/bags-list/src/list/tests.rs diff --git a/frame/balances/bags-list/src/migrations.rs b/frame/bags-list/src/migrations.rs similarity index 100% rename from frame/balances/bags-list/src/migrations.rs rename to frame/bags-list/src/migrations.rs diff --git a/frame/balances/bags-list/src/mock.rs b/frame/bags-list/src/mock.rs similarity index 100% rename from frame/balances/bags-list/src/mock.rs rename to frame/bags-list/src/mock.rs diff --git a/frame/balances/bags-list/src/tests.rs b/frame/bags-list/src/tests.rs similarity index 100% rename from frame/balances/bags-list/src/tests.rs rename to frame/bags-list/src/tests.rs diff --git a/frame/balances/bags-list/src/weights.rs b/frame/bags-list/src/weights.rs similarity index 100% rename from frame/balances/bags-list/src/weights.rs rename to frame/bags-list/src/weights.rs From a00bd76d34d80d52179d928783d1fec6e321e966 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Thu, 25 May 2023 12:07:53 +0530 Subject: [PATCH 19/55] Minor update --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 90940403d6ee2..357cc4406671e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,6 @@ members = [ "frame/authorship", "frame/babe", "frame/bags-list", - "frame/bags-list/src/bags-list", "frame/bags-list/fuzzer", "frame/bags-list/remote-tests", "frame/balances", From 2dda8bfd2496f573834a5fa5b37406c00611a92c Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 10:07:53 +0530 Subject: [PATCH 20/55] Addresses review comment --- frame/examples/split/README.md | 239 +-------------------------------- frame/support/src/lib.rs | 1 - 2 files changed, 3 insertions(+), 237 deletions(-) diff --git a/frame/examples/split/README.md b/frame/examples/split/README.md index 2af50573f8075..ddf8d2a9b1c8f 100644 --- a/frame/examples/split/README.md +++ b/frame/examples/split/README.md @@ -1,240 +1,7 @@ -# Basic Example Pallet +# Basic Example For Splitting A Pallet +A simple example of a FRAME pallet demonstrating the ability to split sections across multiple files. - -The Example: A simple example of a FRAME pallet demonstrating -concepts, APIs and structures common to most FRAME runtimes. - -Run `cargo doc --package pallet-example-basic --open` to view this pallet's documentation. - -**This pallet serves as an example and is not meant to be used in production.** - -### Documentation Guidelines: - - - -
    -
  • Documentation comments (i.e. /// comment) - should - accompany pallet functions and be restricted to the pallet interface, - not the internals of the pallet implementation. Only state inputs, - outputs, and a brief description that mentions whether calling it - requires root, but without repeating the source code details. - Capitalize the first word of each documentation comment and end it with - a full stop. See - Generic example of annotating source code with documentation comments
    • -
  • Self-documenting code - Try to refactor code to be self-documenting.
    • -
  • Code comments - Supplement complex code with a brief explanation, not every line of code.
    • -
  • Identifiers - surround by backticks (i.e. INHERENT_IDENTIFIER, InherentType, - u64)
    • -
  • Usage scenarios - should be simple doctests. The compiler should ensure they stay valid.
    • -
  • Extended tutorials - should be moved to external files and refer to.
    • - -
  • Mandatory - include all of the sections/subsections where MUST is specified.
    • -
  • Optional - optionally include sections/subsections where CAN is specified.
    • -
- -### Documentation Template:
- -Copy and paste this template from frame/examples/basic/src/lib.rs into file -`frame//src/lib.rs` of your own custom pallet and complete it. -

-// Add heading with custom pallet name
-
-\#  Pallet
-
-// Add simple description
-
-// Include the following links that shows what trait needs to be implemented to use the pallet
-// and the supported dispatchables that are documented in the Call enum.
-
-- \[`::Config`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/trait.Config.html)
-- \[`Call`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/enum.Call.html)
-- \[`Module`](https://docs.rs/pallet-example-basic/latest/pallet_example_basic/struct.Module.html)
-
-\## Overview
-
-
-// Short description of pallet's purpose.
-// Links to Traits that should be implemented.
-// What this pallet is for.
-// What functionality the pallet provides.
-// When to use the pallet (use case examples).
-// How it is used.
-// Inputs it uses and the source of each input.
-// Outputs it produces.
-
-
-
-
-\## Terminology
-
-// Add terminology used in the custom pallet. Include concepts, storage items, or actions that you think
-// deserve to be noted to give context to the rest of the documentation or pallet usage. The author needs to
-// use some judgment about what is included. We don't want a list of every storage item nor types - the user
-// can go to the code for that. For example, "transfer fee" is obvious and should not be included, but
-// "free balance" and "reserved balance" should be noted to give context to the pallet.
-// Please do not link to outside resources. The reference docs should be the ultimate source of truth.
-
-
-
-\## Goals
-
-// Add goals that the custom pallet is designed to achieve.
-
-
-
-\### Scenarios
-
-
-
-\#### 
-
-// Describe requirements prior to interacting with the custom pallet.
-// Describe the process of interacting with the custom pallet for this scenario and public API functions used.
-
-\## Interface
-
-\### Supported Origins
-
-// What origins are used and supported in this pallet (root, signed, none)
-// i.e. root when \`ensure_root\` used
-// i.e. none when \`ensure_none\` used
-// i.e. signed when \`ensure_signed\` used
-
-\`inherent\` 
-
-
-
-
-\### Types
-
-// Type aliases. Include any associated types and where the user would typically define them.
-
-\`ExampleType\` 
-
-
-
-// Reference documentation of aspects such as `storageItems` and `dispatchable` functions should only be
-// included in the https://docs.rs Rustdocs for Substrate and not repeated in the README file.
-
-\### Dispatchable Functions
-
-
-
-// A brief description of dispatchable functions and a link to the rustdoc with their actual documentation.
-
-// MUST have link to Call enum
-// MUST have origin information included in function doc
-// CAN have more info up to the user
-
-\### Public Functions
-
-
-
-// A link to the rustdoc and any notes about usage in the pallet, not for specific functions.
-// For example, in the Balances Pallet: "Note that when using the publicly exposed functions,
-// you (the runtime developer) are responsible for implementing any necessary checks
-// (e.g. that the sender is the signer) before calling a function that will affect storage."
-
-
-
-// It is up to the writer of the respective pallet (with respect to how much information to provide).
-
-\#### Public Inspection functions - Immutable (getters)
-
-// Insert a subheading for each getter function signature
-
-\##### \`example_getter_name()\`
-
-// What it returns
-// Why, when, and how often to call it
-// When it could panic or error
-// When safety issues to consider
-
-\#### Public Mutable functions (changing state)
-
-// Insert a subheading for each setter function signature
-
-\##### \`example_setter_name(origin, parameter_name: T::ExampleType)\`
-
-// What state it changes
-// Why, when, and how often to call it
-// When it could panic or error
-// When safety issues to consider
-// What parameter values are valid and why
-
-\### Storage Items
-
-// Explain any storage items included in this pallet
-
-\### Digest Items
-
-// Explain any digest items included in this pallet
-
-\### Inherent Data
-
-// Explain what inherent data (if any) is defined in the pallet and any other related types
-
-\### Events:
-
-// Insert events for this pallet if any
-
-\### Errors:
-
-// Explain what generates errors
-
-\## Usage
-
-// Insert 2-3 examples of usage and code snippets that show how to
-// use  Pallet in a custom pallet.
-
-\### Prerequisites
-
-// Show how to include necessary imports for  and derive
-// your pallet configuration trait with the `INSERT_CUSTOM_PALLET_NAME` trait.
-
-\```rust
-use ;
-
-pub trait Config: ::Config { }
-\```
-
-\### Simple Code Snippet
-
-// Show a simple example (e.g. how to query a public getter function of )
-
-\### Example from FRAME
-
-// Show a usage example in an actual runtime
-
-// See:
-// - Substrate TCR https://github.com/parity-samples/substrate-tcr
-// - Substrate Kitties https://shawntabrizi.github.io/substrate-collectables-workshop/#/
-
-\## Genesis Config
-
-
-
-\## Dependencies
-
-// Dependencies on other FRAME pallets and the genesis config should be mentioned,
-// but not the Rust Standard Library.
-// Genesis configuration modifications that may be made to incorporate this pallet
-// Interaction with other pallets
-
-
-
-\## Related Pallets
-
-// Interaction with other pallets in the form of a bullet point list
-
-\## References
-
-
-
-// Links to reference material, if applicable. For example, Phragmen, W3F research, etc.
-// that the implementation is based on.
-

+Run `cargo doc --package pallet-example-split --open` to view this pallet's documentation. License: MIT-0 diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 0a73271108cb5..0a11595e5764e 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -823,7 +823,6 @@ macro_rules! assert_error_encoded_size { pub use serde::{Deserialize, Serialize}; #[doc(hidden)] -#[cfg(not(no_std))] pub use macro_magic; #[cfg(test)] From d0f83d167a0c7dc33e4ff9c97edd858a3ad39ee9 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 11:04:30 +0530 Subject: [PATCH 21/55] Adds a couple of UI tests. More to be added --- Cargo.lock | 1 + frame/support/test/Cargo.toml | 1 + frame/support/test/tests/split_ui.rs | 36 +++++++++++++++++++ .../test/tests/split_ui/pass/split_valid.rs | 35 ++++++++++++++++++ .../tests/split_ui/section_not_imported.rs | 34 ++++++++++++++++++ 5 files changed, 107 insertions(+) create mode 100644 frame/support/test/tests/split_ui.rs create mode 100644 frame/support/test/tests/split_ui/pass/split_valid.rs create mode 100644 frame/support/test/tests/split_ui/section_not_imported.rs diff --git a/Cargo.lock b/Cargo.lock index 840b290139db6..856efe993334e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2749,6 +2749,7 @@ dependencies = [ "frame-support", "frame-support-test-pallet", "frame-system", + "macro_magic", "parity-scale-codec", "pretty_assertions", "rustversion", diff --git a/frame/support/test/Cargo.toml b/frame/support/test/Cargo.toml index 68411210f9b5c..2ccb83b9a0fd2 100644 --- a/frame/support/test/Cargo.toml +++ b/frame/support/test/Cargo.toml @@ -32,6 +32,7 @@ frame-system = { version = "4.0.0-dev", default-features = false, path = "../../ frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../executive" } # The "std" feature for this pallet is never activated on purpose, in order to test construct_runtime error message test-pallet = { package = "frame-support-test-pallet", default-features = false, path = "pallet" } +macro_magic = "0.3.1" [features] default = ["std"] diff --git a/frame/support/test/tests/split_ui.rs b/frame/support/test/tests/split_ui.rs new file mode 100644 index 0000000000000..07d32fea9e317 --- /dev/null +++ b/frame/support/test/tests/split_ui.rs @@ -0,0 +1,36 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// #[rustversion::attr(not(stable), ignore)] +// #[cfg(not(feature = "disable-ui-tests"))] +#[test] +fn split_ui() { + // // Only run the ui tests when `RUN_UI_TESTS` is set. + // if std::env::var("RUN_UI_TESTS").is_err() { + // return + // } + + // As trybuild is using `cargo check`, we don't need the real WASM binaries. + std::env::set_var("SKIP_WASM_BUILD", "1"); + + // Deny all warnings since we emit warnings as part of a Pallet's UI. + std::env::set_var("RUSTFLAGS", "--deny warnings"); + + let t = trybuild::TestCases::new(); + // t.compile_fail("tests/split_ui/*.rs"); + t.pass("tests/split_ui/pass/*.rs"); +} diff --git a/frame/support/test/tests/split_ui/pass/split_valid.rs b/frame/support/test/tests/split_ui/pass/split_valid.rs new file mode 100644 index 0000000000000..71a22e44aab2d --- /dev/null +++ b/frame/support/test/tests/split_ui/pass/split_valid.rs @@ -0,0 +1,35 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::pallet_macros::*; + +pub use pallet::*; + +#[export_section] +mod storages { + #[pallet::storage] + pub type MyStorageMap = StorageMap<_, _, u32, u64>; +} + +#[import_section(storages)] +#[frame_support::pallet(dev_mode)] +pub mod pallet { + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::call] + impl Pallet { + pub fn my_call(_origin: OriginFor) -> DispatchResult { + MyStorageMap::::insert(1, 2); + Ok(()) + } + } +} + +fn main() { +} \ No newline at end of file diff --git a/frame/support/test/tests/split_ui/section_not_imported.rs b/frame/support/test/tests/split_ui/section_not_imported.rs new file mode 100644 index 0000000000000..dc0a089a57e8c --- /dev/null +++ b/frame/support/test/tests/split_ui/section_not_imported.rs @@ -0,0 +1,34 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::pallet_macros::*; + +pub use pallet::*; + +#[export_section] +mod storages { + #[pallet::storage] + pub type MyStorageMap = StorageMap<_, _, u32, u64>; +} + +#[frame_support::pallet(dev_mode)] +pub mod pallet { + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::call] + impl Pallet { + pub fn my_call(_origin: OriginFor) -> DispatchResult { + MyStorageMap::::insert(1, 2); + Ok(()) + } + } +} + +fn main() { +} \ No newline at end of file From 67cccaeefbd7c9202c344ae15b01b0689b432c19 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 11:08:14 +0530 Subject: [PATCH 22/55] Adds err files --- frame/support/test/tests/split_ui.rs | 2 +- .../test/tests/split_ui/no_section_found.rs | 29 +++++++++++++++++++ .../tests/split_ui/no_section_found.stderr | 13 +++++++++ .../split_ui/section_not_imported.stderr | 8 +++++ 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 frame/support/test/tests/split_ui/no_section_found.rs create mode 100644 frame/support/test/tests/split_ui/no_section_found.stderr create mode 100644 frame/support/test/tests/split_ui/section_not_imported.stderr diff --git a/frame/support/test/tests/split_ui.rs b/frame/support/test/tests/split_ui.rs index 07d32fea9e317..35be76057b86d 100644 --- a/frame/support/test/tests/split_ui.rs +++ b/frame/support/test/tests/split_ui.rs @@ -31,6 +31,6 @@ fn split_ui() { std::env::set_var("RUSTFLAGS", "--deny warnings"); let t = trybuild::TestCases::new(); - // t.compile_fail("tests/split_ui/*.rs"); + t.compile_fail("tests/split_ui/*.rs"); t.pass("tests/split_ui/pass/*.rs"); } diff --git a/frame/support/test/tests/split_ui/no_section_found.rs b/frame/support/test/tests/split_ui/no_section_found.rs new file mode 100644 index 0000000000000..b44a7e9015206 --- /dev/null +++ b/frame/support/test/tests/split_ui/no_section_found.rs @@ -0,0 +1,29 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::pallet_macros::*; + +pub use pallet::*; + +#[import_section(storages_dev)] +#[frame_support::pallet(dev_mode)] +pub mod pallet { + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::call] + impl Pallet { + pub fn my_call(_origin: OriginFor) -> DispatchResult { + MyStorageMap::::insert(1, 2); + Ok(()) + } + } +} + +fn main() { +} \ No newline at end of file diff --git a/frame/support/test/tests/split_ui/no_section_found.stderr b/frame/support/test/tests/split_ui/no_section_found.stderr new file mode 100644 index 0000000000000..05c38c904f97d --- /dev/null +++ b/frame/support/test/tests/split_ui/no_section_found.stderr @@ -0,0 +1,13 @@ +error[E0432]: unresolved import `pallet` + --> tests/split_ui/no_section_found.rs:5:9 + | +5 | pub use pallet::*; + | ^^^^^^ help: a similar path exists: `test_pallet::pallet` + +error: cannot find macro `__export_tokens_tt_storages_dev` in this scope + --> tests/split_ui/no_section_found.rs:7:1 + | +7 | #[import_section(storages_dev)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + = note: this error originates in the macro `::macro_magic::forward_tokens` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/split_ui/section_not_imported.stderr b/frame/support/test/tests/split_ui/section_not_imported.stderr new file mode 100644 index 0000000000000..41ac2a5f58d25 --- /dev/null +++ b/frame/support/test/tests/split_ui/section_not_imported.stderr @@ -0,0 +1,8 @@ +error[E0433]: failed to resolve: use of undeclared type `MyStorageMap` + --> tests/split_ui/section_not_imported.rs:27:4 + | +27 | MyStorageMap::::insert(1, 2); + | ^^^^^^^^^^^^ + | | + | use of undeclared type `MyStorageMap` + | help: a struct with a similar name exists: `StorageMap` From fb8d770a8e66cd25e82d52723bb9b9bf952885ac Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 11:45:06 +0530 Subject: [PATCH 23/55] Adds test for no pallet --- frame/support/procedural/src/lib.rs | 5 +++++ .../tests/split_ui/import_without_pallet.rs | 17 +++++++++++++++++ .../tests/split_ui/import_without_pallet.stderr | 15 +++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 frame/support/test/tests/split_ui/import_without_pallet.rs create mode 100644 frame/support/test/tests/split_ui/import_without_pallet.stderr diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 5f96d17100fd3..a96701a5af7b0 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1439,6 +1439,11 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let foreign_mod = parse_macro_input!(attr as ItemMod); let mut internal_mod = parse_macro_input!(tokens as ItemMod); + let err = syn::Error::new_spanned(internal_mod.clone(), "A section can only be imported under a pallet macro") + .to_compile_error().into(); + + if internal_mod.attrs.len() == 0 { return err; } + if let Some(ref mut content) = internal_mod.content { if let Some(foreign_content) = foreign_mod.content { content.1.extend(foreign_content.1); diff --git a/frame/support/test/tests/split_ui/import_without_pallet.rs b/frame/support/test/tests/split_ui/import_without_pallet.rs new file mode 100644 index 0000000000000..e4b9008745f46 --- /dev/null +++ b/frame/support/test/tests/split_ui/import_without_pallet.rs @@ -0,0 +1,17 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::pallet_macros::*; + +#[export_section] +mod storages { + #[pallet::storage] + pub type MyStorageMap = StorageMap<_, _, u32, u64>; +} + +#[import_section(storages)] +pub mod pallet { + +} + +fn main() { +} \ No newline at end of file diff --git a/frame/support/test/tests/split_ui/import_without_pallet.stderr b/frame/support/test/tests/split_ui/import_without_pallet.stderr new file mode 100644 index 0000000000000..62923f4858984 --- /dev/null +++ b/frame/support/test/tests/split_ui/import_without_pallet.stderr @@ -0,0 +1,15 @@ +error: A section can only be imported under a pallet macro + --> tests/split_ui/import_without_pallet.rs:5:1 + | +5 | / #[export_section] +6 | | mod storages { +7 | | #[pallet::storage] +8 | | pub type MyStorageMap = StorageMap<_, _, u32, u64>; +9 | | } +10 | | +11 | | #[import_section(storages)] + | |_--------------------------^ + | | + | in this procedural macro expansion + | + = note: this error originates in the macro `__import_tokens_attr_import_section_inner` which comes from the expansion of the attribute macro `import_section` (in Nightly builds, run with -Z macro-backtrace for more info) From 58ae2a105631e644ac345db7156968927a0810f8 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 11:54:18 +0530 Subject: [PATCH 24/55] Adds doc --- frame/support/procedural/src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index a96701a5af7b0..e533347ebce31 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1425,6 +1425,9 @@ pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { pallet_macro_stub() } +/// An attribute macro that can be attached to a module declaration. Doing so will +/// make it available for import later. +/// #[proc_macro_attribute] pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { @@ -1433,6 +1436,9 @@ pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { } } +/// An attribute macro that can be attached to a module declaration. Doing so will +/// import the content of the section that was exported previously using [`export_section`]. +/// #[import_tokens_attr] #[proc_macro_attribute] pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { From 56888aff5a2b032282f59ebc596aa6d3091975a2 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 12:29:31 +0530 Subject: [PATCH 25/55] Updates versions --- frame/examples/split/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/examples/split/Cargo.toml b/frame/examples/split/Cargo.toml index 6c83919eb38ac..84e470b7a4457 100644 --- a/frame/examples/split/Cargo.toml +++ b/frame/examples/split/Cargo.toml @@ -20,13 +20,13 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" } -sp-io = { version = "7.0.0", default-features = false, path = "../../../primitives/io" } -sp-runtime = { version = "7.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-std = { version = "5.0.0", default-features = false, path = "../../../primitives/std" } +sp-io = { version = "8.0.0", default-features = false, path = "../../../primitives/io" } +sp-runtime = { version = "8.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "6.0.0", default-features = false, path = "../../../primitives/std" } macro_magic = "0.3.1" [dev-dependencies] -sp-core = { version = "7.0.0", default-features = false, path = "../../../primitives/core" } +sp-core = { version = "8.0.0", default-features = false, path = "../../../primitives/core" } [features] default = ["std"] From 7d34a976ae6bc80cbcf8e0d075e18d00a5315b34 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 13:16:12 +0530 Subject: [PATCH 26/55] Adds benchmarking --- frame/examples/split/src/benchmarking.rs | 35 ++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 frame/examples/split/src/benchmarking.rs diff --git a/frame/examples/split/src/benchmarking.rs b/frame/examples/split/src/benchmarking.rs new file mode 100644 index 0000000000000..5a262417629c5 --- /dev/null +++ b/frame/examples/split/src/benchmarking.rs @@ -0,0 +1,35 @@ +//! Benchmarking setup for pallet-template +#![cfg(feature = "runtime-benchmarks")] +use super::*; + +#[allow(unused)] +use crate::Pallet as Template; +use frame_benchmarking::v2::*; +use frame_system::RawOrigin; + +#[benchmarks] +mod benchmarks { + use super::*; + + #[benchmark] + fn do_something() { + let value = 100u32.into(); + let caller: T::AccountId = whitelisted_caller(); + #[extrinsic_call] + do_something(RawOrigin::Signed(caller), value); + + assert_eq!(Something::::get(), Some(value)); + } + + #[benchmark] + fn cause_error() { + Something::::put(100u32); + let caller: T::AccountId = whitelisted_caller(); + #[extrinsic_call] + cause_error(RawOrigin::Signed(caller)); + + assert_eq!(Something::::get(), Some(101u32)); + } + + impl_benchmark_test_suite!(Template, crate::mock::new_test_ext(), crate::mock::Test); +} From 0ce4726028609257175060c3018f1ae329f58852 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 13:27:03 +0530 Subject: [PATCH 27/55] Updates doc link --- frame/support/procedural/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index e533347ebce31..a72d80e12b0e6 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1437,7 +1437,7 @@ pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { } /// An attribute macro that can be attached to a module declaration. Doing so will -/// import the content of the section that was exported previously using [`export_section`]. +/// import the content of the section that was exported previously using [`macro@export_section`]. /// #[import_tokens_attr] #[proc_macro_attribute] From c60473749e3e253a149b349e3ab2e3af3825b579 Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Tue, 30 May 2023 07:59:13 +0000 Subject: [PATCH 28/55] ".git/.scripts/commands/fmt/fmt.sh" --- frame/examples/split/src/lib.rs | 2 +- frame/examples/split/src/storages.rs | 2 +- frame/support/procedural/src/lib.rs | 24 +++++++++++-------- .../procedural/src/pallet/expand/storage.rs | 2 +- frame/support/src/lib.rs | 13 +++++----- 5 files changed, 23 insertions(+), 20 deletions(-) diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index d1779bf59abc3..59077e956a243 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -15,8 +15,8 @@ mod tests; mod benchmarking; mod storages; pub mod weights; -pub use weights::*; use frame_support::pallet_macros::*; +pub use weights::*; #[import_section(storages)] #[frame_support::pallet] diff --git a/frame/examples/split/src/storages.rs b/frame/examples/split/src/storages.rs index bceb750a7143c..5b7f7a4e049bd 100644 --- a/frame/examples/split/src/storages.rs +++ b/frame/examples/split/src/storages.rs @@ -9,4 +9,4 @@ mod storages { // Learn more about declaring storage items: // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items pub type Something = StorageValue<_, u32>; -} \ No newline at end of file +} diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index a72d80e12b0e6..ccf8cb3c2a2af 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -36,13 +36,12 @@ mod storage_alias; mod transactional; mod tt_macro; -use proc_macro::TokenStream; -use quote::quote; use macro_magic::*; -use syn::{parse_macro_input, ItemMod}; +use proc_macro::TokenStream; +use quote::{quote, ToTokens}; use std::{cell::RefCell, str::FromStr}; -use quote::ToTokens; pub(crate) use storage::INHERENT_INSTANCE_NAME; +use syn::{parse_macro_input, ItemMod}; thread_local! { /// A global counter, can be used to generate a relatively unique identifier. @@ -1427,7 +1426,6 @@ pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { /// An attribute macro that can be attached to a module declaration. Doing so will /// make it available for import later. -/// #[proc_macro_attribute] pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { @@ -1438,17 +1436,22 @@ pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { /// An attribute macro that can be attached to a module declaration. Doing so will /// import the content of the section that was exported previously using [`macro@export_section`]. -/// #[import_tokens_attr] #[proc_macro_attribute] pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let foreign_mod = parse_macro_input!(attr as ItemMod); let mut internal_mod = parse_macro_input!(tokens as ItemMod); - let err = syn::Error::new_spanned(internal_mod.clone(), "A section can only be imported under a pallet macro") - .to_compile_error().into(); + let err = syn::Error::new_spanned( + internal_mod.clone(), + "A section can only be imported under a pallet macro", + ) + .to_compile_error() + .into(); - if internal_mod.attrs.len() == 0 { return err; } + if internal_mod.attrs.len() == 0 { + return err + } if let Some(ref mut content) = internal_mod.content { if let Some(foreign_content) = foreign_mod.content { @@ -1458,5 +1461,6 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { quote! { #internal_mod - }.into() + } + .into() } diff --git a/frame/support/procedural/src/pallet/expand/storage.rs b/frame/support/procedural/src/pallet/expand/storage.rs index 89ae8382a38da..1a168ee16f3cc 100644 --- a/frame/support/procedural/src/pallet/expand/storage.rs +++ b/frame/support/procedural/src/pallet/expand/storage.rs @@ -23,10 +23,10 @@ use crate::{ }, }; use itertools::Itertools; +use proc_macro2::Span; use quote::ToTokens; use std::{collections::HashMap, ops::IndexMut}; use syn::spanned::Spanned; -use proc_macro2::Span; /// Generate the prefix_ident related to the storage. /// prefix_ident is used for the prefix struct to be given to storage as first generic param. diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index d1e6a87473344..1a4358ffb5ae7 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2893,16 +2893,15 @@ pub use frame_support_procedural::pallet; /// Contains macro stubs for all of the pallet:: macros pub mod pallet_macros { + #[macro_magic::use_attr] + pub use frame_support_procedural::import_section; pub use frame_support_procedural::{ call_index, compact, composite_enum, config, constant, - disable_frame_system_supertrait_check, error, event, extra_constants, generate_deposit, - generate_store, genesis_build, genesis_config, getter, hooks, inherent, origin, storage, - storage_prefix, storage_version, type_value, unbounded, validate_unsigned, weight, - whitelist_storage, + disable_frame_system_supertrait_check, error, event, export_section, extra_constants, + generate_deposit, generate_store, genesis_build, genesis_config, getter, hooks, inherent, + origin, storage, storage_prefix, storage_version, type_value, unbounded, validate_unsigned, + weight, whitelist_storage, }; - #[macro_magic::use_attr] - pub use frame_support_procedural::import_section; - pub use frame_support_procedural::export_section; } // Generate a macro that will enable/disable code based on `std` feature being active. From b83b88669738ca55e7630da1cd482dadf0a07a59 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 13:34:21 +0530 Subject: [PATCH 29/55] Minor update --- frame/support/test/tests/split_ui.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frame/support/test/tests/split_ui.rs b/frame/support/test/tests/split_ui.rs index 35be76057b86d..14f99b8ecdab1 100644 --- a/frame/support/test/tests/split_ui.rs +++ b/frame/support/test/tests/split_ui.rs @@ -15,14 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -// #[rustversion::attr(not(stable), ignore)] -// #[cfg(not(feature = "disable-ui-tests"))] +#[rustversion::attr(not(stable), ignore)] +#[cfg(not(feature = "disable-ui-tests"))] #[test] fn split_ui() { - // // Only run the ui tests when `RUN_UI_TESTS` is set. - // if std::env::var("RUN_UI_TESTS").is_err() { - // return - // } + // Only run the ui tests when `RUN_UI_TESTS` is set. + if std::env::var("RUN_UI_TESTS").is_err() { + return + } // As trybuild is using `cargo check`, we don't need the real WASM binaries. std::env::set_var("SKIP_WASM_BUILD", "1"); From 4896f97035953b65e1fbae9569a881825e6ccf6c Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 14:07:31 +0530 Subject: [PATCH 30/55] Adds missing changes --- frame/support/procedural/src/lib.rs | 43 ++++++++++++++++++++++++++++- frame/support/src/lib.rs | 4 ++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 84a8b37f92e06..1b22619a7d432 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -42,7 +42,7 @@ use proc_macro::TokenStream; use quote::{quote, ToTokens}; use std::{cell::RefCell, str::FromStr}; pub(crate) use storage::INHERENT_INSTANCE_NAME; -use syn::{parse_macro_input, ItemImpl}; +use syn::{parse_macro_input, ItemImpl, ItemMod}; thread_local! { /// A global counter, can be used to generate a relatively unique identifier. @@ -1738,3 +1738,44 @@ pub fn origin(_: TokenStream, _: TokenStream) -> TokenStream { pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { pallet_macro_stub() } + +/// An attribute macro that can be attached to a module declaration. Doing so will +/// make it available for import later. +#[proc_macro_attribute] +pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { + match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { + Ok(tokens) => tokens.into(), + Err(err) => err.to_compile_error().into(), + } +} + +/// An attribute macro that can be attached to a module declaration. Doing so will +/// import the content of the section that was exported previously using [`macro@export_section`]. +#[import_tokens_attr] +#[proc_macro_attribute] +pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { + let foreign_mod = parse_macro_input!(attr as ItemMod); + let mut internal_mod = parse_macro_input!(tokens as ItemMod); + + let err = syn::Error::new_spanned( + internal_mod.clone(), + "A section can only be imported under a pallet macro", + ) + .to_compile_error() + .into(); + + if internal_mod.attrs.len() == 0 { + return err + } + + if let Some(ref mut content) = internal_mod.content { + if let Some(foreign_content) = foreign_mod.content { + content.1.extend(foreign_content.1); + } + } + + quote! { + #internal_mod + } + .into() +} diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index de87b59a0b943..97058c164ec6a 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2898,9 +2898,11 @@ pub use frame_support_procedural::pallet; /// Contains macro stubs for all of the pallet:: macros pub mod pallet_macros { + #[macro_magic::use_attr] + pub use frame_support_procedural::import_section; pub use frame_support_procedural::{ call_index, compact, composite_enum, config, constant, - disable_frame_system_supertrait_check, error, event, extra_constants, generate_deposit, + disable_frame_system_supertrait_check, error, event, export_section, extra_constants, generate_deposit, generate_store, genesis_build, genesis_config, getter, hooks, inherent, no_default, origin, storage, storage_prefix, storage_version, type_value, unbounded, validate_unsigned, weight, whitelist_storage, From c72d3fcf9e2109925e94ac7720827ff7bcf6454f Mon Sep 17 00:00:00 2001 From: command-bot Date: Tue, 30 May 2023 09:35:58 +0000 Subject: [PATCH 31/55] ".git/.scripts/commands/fmt/fmt.sh" --- frame/support/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 97058c164ec6a..9640c0a336532 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2902,10 +2902,10 @@ pub mod pallet_macros { pub use frame_support_procedural::import_section; pub use frame_support_procedural::{ call_index, compact, composite_enum, config, constant, - disable_frame_system_supertrait_check, error, event, export_section, extra_constants, generate_deposit, - generate_store, genesis_build, genesis_config, getter, hooks, inherent, no_default, origin, - storage, storage_prefix, storage_version, type_value, unbounded, validate_unsigned, weight, - whitelist_storage, + disable_frame_system_supertrait_check, error, event, export_section, extra_constants, + generate_deposit, generate_store, genesis_build, genesis_config, getter, hooks, inherent, + no_default, origin, storage, storage_prefix, storage_version, type_value, unbounded, + validate_unsigned, weight, whitelist_storage, }; } From 8d4c5283dc7f73a54502069e4855408370c2c039 Mon Sep 17 00:00:00 2001 From: gupnik <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 20:36:51 +0530 Subject: [PATCH 32/55] Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson --- frame/support/procedural/src/lib.rs | 33 ++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 1b22619a7d432..385bfcc266875 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1757,15 +1757,32 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let foreign_mod = parse_macro_input!(attr as ItemMod); let mut internal_mod = parse_macro_input!(tokens as ItemMod); - let err = syn::Error::new_spanned( - internal_mod.clone(), - "A section can only be imported under a pallet macro", - ) - .to_compile_error() - .into(); + // check that internal_mod is a pallet module + if internal_mod.attrs.iter().find(|attr| { + if let Some(last_seg) = attr.path().segments.last() { + last_seg.ident == "pallet" + } else { + false + } + }).is_none() { + return Err(Error::new( + internal_mod.span(), + "`#[import_section]` can only be applied to a valid pallet module", + )).to_compile_error.into(); + } - if internal_mod.attrs.len() == 0 { - return err + // check that foreign_mod has `#[pallet_section]` attached to it + if foreign_mod.attrs.iter().find(|attr| { + if let Some(last_seg) = attr.path().segments.last() { + last_seg.ident == "pallet_section" + } else { + false + } + }).is_none() { + return Err(Error::new( + foreign_mod.span(), + "`#[import_section]` can only take a valid pallet section as an argument", + )).to_compile_error().into(); } if let Some(ref mut content) = internal_mod.content { From bdb17e70c4032c0ecf4e29ab0e1799ef505e74e9 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 20:41:50 +0530 Subject: [PATCH 33/55] Addresses review comments --- frame/examples/split/src/storages.rs | 2 +- frame/support/procedural/src/lib.rs | 4 ++-- frame/support/src/lib.rs | 2 +- frame/support/test/tests/split_ui/import_without_pallet.rs | 4 ++-- .../support/test/tests/split_ui/import_without_pallet.stderr | 2 +- frame/support/test/tests/split_ui/pass/split_valid.rs | 4 ++-- frame/support/test/tests/split_ui/section_not_imported.rs | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/frame/examples/split/src/storages.rs b/frame/examples/split/src/storages.rs index 5b7f7a4e049bd..e210f6d0b22c6 100644 --- a/frame/examples/split/src/storages.rs +++ b/frame/examples/split/src/storages.rs @@ -1,6 +1,6 @@ use frame_support::pallet_macros::*; -#[export_section] +#[pallet_section] mod storages { // The pallet's runtime storage items. // https://docs.substrate.io/main-docs/build/runtime-storage/ diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 1b22619a7d432..ff1e06f087a5a 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1742,7 +1742,7 @@ pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { /// An attribute macro that can be attached to a module declaration. Doing so will /// make it available for import later. #[proc_macro_attribute] -pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { +pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { Ok(tokens) => tokens.into(), Err(err) => err.to_compile_error().into(), @@ -1750,7 +1750,7 @@ pub fn export_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { } /// An attribute macro that can be attached to a module declaration. Doing so will -/// import the content of the section that was exported previously using [`macro@export_section`]. +/// import the content of the section that was exported previously using [`macro@pallet_section`]. #[import_tokens_attr] #[proc_macro_attribute] pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 97058c164ec6a..4ee5d836b627b 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2902,7 +2902,7 @@ pub mod pallet_macros { pub use frame_support_procedural::import_section; pub use frame_support_procedural::{ call_index, compact, composite_enum, config, constant, - disable_frame_system_supertrait_check, error, event, export_section, extra_constants, generate_deposit, + disable_frame_system_supertrait_check, error, event, pallet_section, extra_constants, generate_deposit, generate_store, genesis_build, genesis_config, getter, hooks, inherent, no_default, origin, storage, storage_prefix, storage_version, type_value, unbounded, validate_unsigned, weight, whitelist_storage, diff --git a/frame/support/test/tests/split_ui/import_without_pallet.rs b/frame/support/test/tests/split_ui/import_without_pallet.rs index e4b9008745f46..874a92e461098 100644 --- a/frame/support/test/tests/split_ui/import_without_pallet.rs +++ b/frame/support/test/tests/split_ui/import_without_pallet.rs @@ -2,7 +2,7 @@ use frame_support::pallet_macros::*; -#[export_section] +#[pallet_section] mod storages { #[pallet::storage] pub type MyStorageMap = StorageMap<_, _, u32, u64>; @@ -14,4 +14,4 @@ pub mod pallet { } fn main() { -} \ No newline at end of file +} diff --git a/frame/support/test/tests/split_ui/import_without_pallet.stderr b/frame/support/test/tests/split_ui/import_without_pallet.stderr index 62923f4858984..0f714c1da7501 100644 --- a/frame/support/test/tests/split_ui/import_without_pallet.stderr +++ b/frame/support/test/tests/split_ui/import_without_pallet.stderr @@ -1,7 +1,7 @@ error: A section can only be imported under a pallet macro --> tests/split_ui/import_without_pallet.rs:5:1 | -5 | / #[export_section] +5 | / #[pallet_section] 6 | | mod storages { 7 | | #[pallet::storage] 8 | | pub type MyStorageMap = StorageMap<_, _, u32, u64>; diff --git a/frame/support/test/tests/split_ui/pass/split_valid.rs b/frame/support/test/tests/split_ui/pass/split_valid.rs index 71a22e44aab2d..a1342f49399da 100644 --- a/frame/support/test/tests/split_ui/pass/split_valid.rs +++ b/frame/support/test/tests/split_ui/pass/split_valid.rs @@ -4,7 +4,7 @@ use frame_support::pallet_macros::*; pub use pallet::*; -#[export_section] +#[pallet_section] mod storages { #[pallet::storage] pub type MyStorageMap = StorageMap<_, _, u32, u64>; @@ -32,4 +32,4 @@ pub mod pallet { } fn main() { -} \ No newline at end of file +} diff --git a/frame/support/test/tests/split_ui/section_not_imported.rs b/frame/support/test/tests/split_ui/section_not_imported.rs index dc0a089a57e8c..bcabf66256771 100644 --- a/frame/support/test/tests/split_ui/section_not_imported.rs +++ b/frame/support/test/tests/split_ui/section_not_imported.rs @@ -4,7 +4,7 @@ use frame_support::pallet_macros::*; pub use pallet::*; -#[export_section] +#[pallet_section] mod storages { #[pallet::storage] pub type MyStorageMap = StorageMap<_, _, u32, u64>; @@ -31,4 +31,4 @@ pub mod pallet { } fn main() { -} \ No newline at end of file +} From eead306fbf174b3ef07101d1ca69fc98212d6d9c Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 30 May 2023 22:01:17 +0530 Subject: [PATCH 34/55] Addresses review comments --- frame/support/procedural/src/lib.rs | 32 ++++++------------- .../split_ui/import_without_pallet.stderr | 2 +- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index f754e1ec1d77e..407cafb90b5cd 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -42,7 +42,7 @@ use proc_macro::TokenStream; use quote::{quote, ToTokens}; use std::{cell::RefCell, str::FromStr}; pub(crate) use storage::INHERENT_INSTANCE_NAME; -use syn::{parse_macro_input, ItemImpl, ItemMod}; +use syn::{parse_macro_input, Error, ItemImpl, ItemMod}; thread_local! { /// A global counter, can be used to generate a relatively unique identifier. @@ -1759,30 +1759,16 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { // check that internal_mod is a pallet module if internal_mod.attrs.iter().find(|attr| { - if let Some(last_seg) = attr.path().segments.last() { - last_seg.ident == "pallet" - } else { - false - } + if let Some(last_seg) = attr.path().segments.last() { + last_seg.ident == "pallet" + } else { + false + } }).is_none() { - return Err(Error::new( - internal_mod.span(), + return Error::new( + internal_mod.ident.span(), "`#[import_section]` can only be applied to a valid pallet module", - )).to_compile_error.into(); - } - - // check that foreign_mod has `#[pallet_section]` attached to it - if foreign_mod.attrs.iter().find(|attr| { - if let Some(last_seg) = attr.path().segments.last() { - last_seg.ident == "pallet_section" - } else { - false - } - }).is_none() { - return Err(Error::new( - foreign_mod.span(), - "`#[import_section]` can only take a valid pallet section as an argument", - )).to_compile_error().into(); + ).to_compile_error().into(); } if let Some(ref mut content) = internal_mod.content { diff --git a/frame/support/test/tests/split_ui/import_without_pallet.stderr b/frame/support/test/tests/split_ui/import_without_pallet.stderr index 0f714c1da7501..4a1b8d44fe88f 100644 --- a/frame/support/test/tests/split_ui/import_without_pallet.stderr +++ b/frame/support/test/tests/split_ui/import_without_pallet.stderr @@ -1,4 +1,4 @@ -error: A section can only be imported under a pallet macro +error: `#[import_section]` can only be applied to a valid pallet module --> tests/split_ui/import_without_pallet.rs:5:1 | 5 | / #[pallet_section] From 5f950ad080a121f77a353d1e8ec3734b5e067958 Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Tue, 30 May 2023 16:33:30 +0000 Subject: [PATCH 35/55] ".git/.scripts/commands/fmt/fmt.sh" --- frame/support/procedural/src/lib.rs | 29 ++++++++++++++++++----------- frame/support/src/lib.rs | 6 +++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 407cafb90b5cd..0282b4cb606b8 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1758,17 +1758,24 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let mut internal_mod = parse_macro_input!(tokens as ItemMod); // check that internal_mod is a pallet module - if internal_mod.attrs.iter().find(|attr| { - if let Some(last_seg) = attr.path().segments.last() { - last_seg.ident == "pallet" - } else { - false - } - }).is_none() { - return Error::new( - internal_mod.ident.span(), - "`#[import_section]` can only be applied to a valid pallet module", - ).to_compile_error().into(); + if internal_mod + .attrs + .iter() + .find(|attr| { + if let Some(last_seg) = attr.path().segments.last() { + last_seg.ident == "pallet" + } else { + false + } + }) + .is_none() + { + return Error::new( + internal_mod.ident.span(), + "`#[import_section]` can only be applied to a valid pallet module", + ) + .to_compile_error() + .into() } if let Some(ref mut content) = internal_mod.content { diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index a7f0ec8cf93f9..409b8749e01d9 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2902,9 +2902,9 @@ pub mod pallet_macros { pub use frame_support_procedural::import_section; pub use frame_support_procedural::{ call_index, compact, composite_enum, config, constant, - disable_frame_system_supertrait_check, error, event, pallet_section, extra_constants, - generate_deposit, generate_store, genesis_build, genesis_config, getter, hooks, inherent, - no_default, origin, storage, storage_prefix, storage_version, type_value, unbounded, + disable_frame_system_supertrait_check, error, event, extra_constants, generate_deposit, + generate_store, genesis_build, genesis_config, getter, hooks, inherent, no_default, origin, + pallet_section, storage, storage_prefix, storage_version, type_value, unbounded, validate_unsigned, weight, whitelist_storage, }; } From 6fc2ff900524476230ae1d2d87915f86f3db3bf2 Mon Sep 17 00:00:00 2001 From: gupnik <17176722+gupnik@users.noreply.github.com> Date: Wed, 31 May 2023 08:41:32 +0530 Subject: [PATCH 36/55] Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson --- frame/support/procedural/src/lib.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 0282b4cb606b8..69bfd5a6359ed 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1739,8 +1739,25 @@ pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { pallet_macro_stub() } -/// An attribute macro that can be attached to a module declaration. Doing so will -/// make it available for import later. +/// Can be attached to a module. Doing so will declare that module as importable into a pallet +/// via [`#[import_section]`](`macro@import_section`). +/// +/// Note that sections are imported by their module name/ident, and should be referred to by +/// their _full path_ from the perspective of the target pallet. Do not attempt to make use +/// of `use` statements to bring pallet sections into scope, as this will not work (unless +/// you do so as part of a wildcard import, in which case it will work). +/// +/// ## Naming Logistics +/// +/// Also note that because of how `#[pallet_section]` works, pallet section names must be +/// globally unique _within the crate in which they are defined_. For more information on +/// why this must be the case, see macro_magic's +/// [`#[export_tokens]`](https://docs.rs/macro_magic/latest/macro_magic/attr.export_tokens.html) macro. +/// +/// Optionally, you may provide an argument to `#[pallet_section]` such as +/// `#[pallet_section(some_ident)]`, in the event that there is another pallet section in +/// same crate with the same ident/name. The ident you specify can then be used instead of +/// the module's ident name when you go to import it via `#[import_section]`. #[proc_macro_attribute] pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { From a230ef125f6a5f9d151e63f27d6841512ed6e3cc Mon Sep 17 00:00:00 2001 From: gupnik <17176722+gupnik@users.noreply.github.com> Date: Wed, 31 May 2023 08:42:28 +0530 Subject: [PATCH 37/55] Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson --- frame/support/procedural/src/lib.rs | 33 ++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 69bfd5a6359ed..3010274fab740 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1767,7 +1767,38 @@ pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { } /// An attribute macro that can be attached to a module declaration. Doing so will -/// import the content of the section that was exported previously using [`macro@pallet_section`]. +/// Imports the contents of the specified external pallet section that was defined +/// previously using [`#[pallet_section]`](`macro@pallet_section`). +/// +/// ## Example +/// ```ignore +/// #[import_section(some_section)] +/// #[pallet] +/// pub mod pallet { +/// // ... +/// } +/// ``` +/// where `some_section` was defined elsewhere via: +/// ```ignore +/// #[pallet_section] +/// pub mod some_section { +/// // ... +/// } +/// ``` +/// +/// This will result in the contents of `some_section` being _verbatim_ imported into +/// the pallet above. Note that since the tokens for `some_section` are essentially +/// copy-pasted into the target pallet, you cannot refer to imports that don't also +/// exist in the target pallet, but this is easily resolved by including all relevant +/// `use` statements within your pallet section, so they are imported as well, or by +/// otherwise ensuring that you have the same imports on the target pallet. +/// +/// It is perfectly permissible to import multiple pallet sections into the same pallet, +/// which can be done by having multiple `#[import_section(something)]` attributes +/// attached to the pallet. +/// +/// Note that sections are imported by their module name/ident, and should be referred to by +/// their _full path_ from the perspective of the target pallet. #[import_tokens_attr] #[proc_macro_attribute] pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { From 39be12e51832ae86dfaccd6e32d4d098ef102ef5 Mon Sep 17 00:00:00 2001 From: gupnik <17176722+gupnik@users.noreply.github.com> Date: Wed, 31 May 2023 08:43:01 +0530 Subject: [PATCH 38/55] Update frame/support/procedural/src/lib.rs Co-authored-by: Sam Johnson --- frame/support/procedural/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 3010274fab740..907fa912945e6 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1760,6 +1760,11 @@ pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { /// the module's ident name when you go to import it via `#[import_section]`. #[proc_macro_attribute] pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { + let tokens_clone = tokens.clone(); + // ensure this can only be attached to a module + let _mod = parse_macro_input!(tokens_clone as ItemMod); + + // use macro_magic's export_tokens as the internal implementation otherwise match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { Ok(tokens) => tokens.into(), Err(err) => err.to_compile_error().into(), From 3db7f3fc3221fc6f289f65909a986b23ab36da03 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 31 May 2023 09:48:42 +0530 Subject: [PATCH 39/55] Adds UI test for disambiguation --- .../pass/split_valid_disambiguation.rs | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs diff --git a/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs b/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs new file mode 100644 index 0000000000000..7e3c48f11cafe --- /dev/null +++ b/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs @@ -0,0 +1,51 @@ +#![cfg_attr(not(feature = "std"), no_std)] + +use frame_support::pallet_macros::*; + +pub use pallet::*; + +mod first { + use super::*; + + #[pallet_section] + mod storages { + #[pallet::storage] + pub type MyStorageMap = StorageMap<_, _, u32, u64>; + } +} + +mod second { + use super::*; + + #[pallet_section(storages2)] + mod storages { + #[pallet::storage] + pub type MyStorageMap2 = StorageMap<_, _, u32, u64>; + } +} + +#[import_section(storages)] +#[import_section(storages2)] +#[frame_support::pallet(dev_mode)] +pub mod pallet { + use frame_support::pallet_prelude::*; + use frame_system::pallet_prelude::*; + + #[pallet::pallet] + pub struct Pallet(_); + + #[pallet::config] + pub trait Config: frame_system::Config {} + + #[pallet::call] + impl Pallet { + pub fn my_call(_origin: OriginFor) -> DispatchResult { + MyStorageMap::::insert(1, 2); + MyStorageMap2::::insert(1, 2); + Ok(()) + } + } +} + +fn main() { +} From d14fb1a251393f2da78807f2dfb6a594f297f14e Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Wed, 31 May 2023 04:20:29 +0000 Subject: [PATCH 40/55] ".git/.scripts/commands/fmt/fmt.sh" --- frame/support/procedural/src/lib.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 907fa912945e6..9b7e4205966e2 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1757,14 +1757,14 @@ pub fn composite_enum(_: TokenStream, _: TokenStream) -> TokenStream { /// Optionally, you may provide an argument to `#[pallet_section]` such as /// `#[pallet_section(some_ident)]`, in the event that there is another pallet section in /// same crate with the same ident/name. The ident you specify can then be used instead of -/// the module's ident name when you go to import it via `#[import_section]`. +/// the module's ident name when you go to import it via `#[import_section]`. #[proc_macro_attribute] pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { - let tokens_clone = tokens.clone(); - // ensure this can only be attached to a module - let _mod = parse_macro_input!(tokens_clone as ItemMod); - - // use macro_magic's export_tokens as the internal implementation otherwise + let tokens_clone = tokens.clone(); + // ensure this can only be attached to a module + let _mod = parse_macro_input!(tokens_clone as ItemMod); + + // use macro_magic's export_tokens as the internal implementation otherwise match macro_magic::mm_core::export_tokens_internal(attr, tokens, false) { Ok(tokens) => tokens.into(), Err(err) => err.to_compile_error().into(), @@ -1772,7 +1772,7 @@ pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { } /// An attribute macro that can be attached to a module declaration. Doing so will -/// Imports the contents of the specified external pallet section that was defined +/// Imports the contents of the specified external pallet section that was defined /// previously using [`#[pallet_section]`](`macro@pallet_section`). /// /// ## Example From 8c44128b99746273d210e160b0f3d381f9153353 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 31 May 2023 10:06:24 +0530 Subject: [PATCH 41/55] Makes clippy happy --- frame/support/procedural/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 9b7e4205966e2..3346e057b04de 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1811,17 +1811,16 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let mut internal_mod = parse_macro_input!(tokens as ItemMod); // check that internal_mod is a pallet module - if internal_mod + if !internal_mod .attrs .iter() - .find(|attr| { + .any(|attr| { if let Some(last_seg) = attr.path().segments.last() { last_seg.ident == "pallet" } else { false } }) - .is_none() { return Error::new( internal_mod.ident.span(), From 5e7273760c85b9f9c6b42ccc5fc00a69bbdd2008 Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Wed, 31 May 2023 04:41:01 +0000 Subject: [PATCH 42/55] ".git/.scripts/commands/fmt/fmt.sh" --- frame/support/procedural/src/lib.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 3346e057b04de..fe2d04cd1ef2c 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1811,17 +1811,13 @@ pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let mut internal_mod = parse_macro_input!(tokens as ItemMod); // check that internal_mod is a pallet module - if !internal_mod - .attrs - .iter() - .any(|attr| { - if let Some(last_seg) = attr.path().segments.last() { - last_seg.ident == "pallet" - } else { - false - } - }) - { + if !internal_mod.attrs.iter().any(|attr| { + if let Some(last_seg) = attr.path().segments.last() { + last_seg.ident == "pallet" + } else { + false + } + }) { return Error::new( internal_mod.ident.span(), "`#[import_section]` can only be applied to a valid pallet module", From ac9fd8d5416667f676d6c300a9048ed4b5df9435 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 31 May 2023 20:19:48 +0530 Subject: [PATCH 43/55] Fixes frame support test --- ...re_span_are_ok_on_wrong_gen_unnamed.stderr | 138 +++++++++--------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr index 1c010d662d07a..77093080ee760 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr @@ -55,76 +55,80 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `PartialStorageInfoTrait` error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 - | -21 | #[pallet::storage] - | ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` - | - = help: the following other types implement trait `TypeInfo`: - &T - &mut T - () - (A, B) - (A, B, C) - (A, B, C, D) - (A, B, C, D, E) - (A, B, C, D, E, F) - and $N others - = note: required for `Bar` to implement `StaticTypeInfo` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 + | +1 | #[frame_support::pallet] + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` + | + = help: the following other types implement trait `TypeInfo`: + &T + &mut T + () + (A, B) + (A, B, C) + (A, B, C, D) + (A, B, C, D, E) + (A, B, C, D, E, F) + and $N others + = note: required for `Bar` to implement `StaticTypeInfo` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 - | -21 | #[pallet::storage] - | ^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` - | - = help: the following other types implement trait `WrapperTypeDecode`: - Arc - Box - Rc - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes - = note: required for `Bar` to implement `Decode` - = note: required for `Bar` to implement `FullCodec` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 + | +1 | #[frame_support::pallet] + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` + | + = help: the following other types implement trait `WrapperTypeDecode`: + Arc + Box + Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes + = note: required for `Bar` to implement `Decode` + = note: required for `Bar` to implement `FullCodec` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 - | -21 | #[pallet::storage] - | ^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` - | - = help: the following other types implement trait `EncodeLike`: - <&&T as EncodeLike> - <&T as EncodeLike> - <&T as EncodeLike> - <&[(K, V)] as EncodeLike>> - <&[(T,)] as EncodeLike>> - <&[(T,)] as EncodeLike>> - <&[(T,)] as EncodeLike>> - <&[T] as EncodeLike>> - and $N others - = note: required for `Bar` to implement `FullEncode` - = note: required for `Bar` to implement `FullCodec` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 + | +1 | #[frame_support::pallet] + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` + | + = help: the following other types implement trait `EncodeLike`: + <&&T as EncodeLike> + <&T as EncodeLike> + <&T as EncodeLike> + <&[(K, V)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[T] as EncodeLike>> + and $N others + = note: required for `Bar` to implement `FullEncode` + = note: required for `Bar` to implement `FullCodec` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 - | -21 | #[pallet::storage] - | ^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` - | - = help: the following other types implement trait `WrapperTypeEncode`: - &T - &mut T - Arc - Box - Cow<'a, T> - Rc - Vec - bytes::bytes::Bytes - and $N others - = note: required for `Bar` to implement `Encode` - = note: required for `Bar` to implement `FullEncode` - = note: required for `Bar` to implement `FullCodec` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 + | +1 | #[frame_support::pallet] + | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` + | + = help: the following other types implement trait `WrapperTypeEncode`: + &T + &mut T + Arc + Box + Cow<'a, T> + Rc + Vec + bytes::bytes::Bytes + and $N others + = note: required for `Bar` to implement `Encode` + = note: required for `Bar` to implement `FullEncode` + = note: required for `Bar` to implement `FullCodec` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` + = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) From dc297c14eed240a4b4f5439cb077f3bbfce71339 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:17:53 +0530 Subject: [PATCH 44/55] Fixes frame support test --- .../procedural/src/pallet/expand/storage.rs | 5 +- ...re_span_are_ok_on_wrong_gen_unnamed.stderr | 138 +++++++++--------- 2 files changed, 68 insertions(+), 75 deletions(-) diff --git a/frame/support/procedural/src/pallet/expand/storage.rs b/frame/support/procedural/src/pallet/expand/storage.rs index 1a168ee16f3cc..253b429bb6eb3 100644 --- a/frame/support/procedural/src/pallet/expand/storage.rs +++ b/frame/support/procedural/src/pallet/expand/storage.rs @@ -23,7 +23,6 @@ use crate::{ }, }; use itertools::Itertools; -use proc_macro2::Span; use quote::ToTokens; use std::{collections::HashMap, ops::IndexMut}; use syn::spanned::Spanned; @@ -411,9 +410,7 @@ pub fn expand_storages(def: &mut Def) -> proc_macro2::TokenStream { let cfg_attrs = &storage.cfg_attrs; - // Since we are using `entries` defined later, we need to specify `call_site` to disable - // macro hygiene. - quote::quote_spanned!(Span::call_site() => + quote::quote_spanned!(storage.attr_span => #(#cfg_attrs)* { <#full_ident as #frame_support::storage::StorageEntryMetadataBuilder>::build_metadata( diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr index 77093080ee760..1c010d662d07a 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr @@ -55,80 +55,76 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `PartialStorageInfoTrait` error[E0277]: the trait bound `Bar: TypeInfo` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 - | -1 | #[frame_support::pallet] - | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` - | - = help: the following other types implement trait `TypeInfo`: - &T - &mut T - () - (A, B) - (A, B, C) - (A, B, C, D) - (A, B, C, D, E) - (A, B, C, D, E, F) - and $N others - = note: required for `Bar` to implement `StaticTypeInfo` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` - = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 + | +21 | #[pallet::storage] + | ^^^^^^^ the trait `TypeInfo` is not implemented for `Bar` + | + = help: the following other types implement trait `TypeInfo`: + &T + &mut T + () + (A, B) + (A, B, C) + (A, B, C, D) + (A, B, C, D, E) + (A, B, C, D, E, F) + and $N others + = note: required for `Bar` to implement `StaticTypeInfo` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 - | -1 | #[frame_support::pallet] - | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` - | - = help: the following other types implement trait `WrapperTypeDecode`: - Arc - Box - Rc - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes - = note: required for `Bar` to implement `Decode` - = note: required for `Bar` to implement `FullCodec` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` - = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 + | +21 | #[pallet::storage] + | ^^^^^^^ the trait `WrapperTypeDecode` is not implemented for `Bar` + | + = help: the following other types implement trait `WrapperTypeDecode`: + Arc + Box + Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes + = note: required for `Bar` to implement `Decode` + = note: required for `Bar` to implement `FullCodec` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 - | -1 | #[frame_support::pallet] - | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` - | - = help: the following other types implement trait `EncodeLike`: - <&&T as EncodeLike> - <&T as EncodeLike> - <&T as EncodeLike> - <&[(K, V)] as EncodeLike>> - <&[(T,)] as EncodeLike>> - <&[(T,)] as EncodeLike>> - <&[(T,)] as EncodeLike>> - <&[T] as EncodeLike>> - and $N others - = note: required for `Bar` to implement `FullEncode` - = note: required for `Bar` to implement `FullCodec` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` - = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 + | +21 | #[pallet::storage] + | ^^^^^^^ the trait `EncodeLike` is not implemented for `Bar` + | + = help: the following other types implement trait `EncodeLike`: + <&&T as EncodeLike> + <&T as EncodeLike> + <&T as EncodeLike> + <&[(K, V)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[(T,)] as EncodeLike>> + <&[T] as EncodeLike>> + and $N others + = note: required for `Bar` to implement `FullEncode` + = note: required for `Bar` to implement `FullCodec` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied - --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:1:1 - | -1 | #[frame_support::pallet] - | ^^^^^^^^^^^^^^^^^^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` - | - = help: the following other types implement trait `WrapperTypeEncode`: - &T - &mut T - Arc - Box - Cow<'a, T> - Rc - Vec - bytes::bytes::Bytes - and $N others - = note: required for `Bar` to implement `Encode` - = note: required for `Bar` to implement `FullEncode` - = note: required for `Bar` to implement `FullCodec` - = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` - = note: this error originates in the attribute macro `frame_support::pallet` (in Nightly builds, run with -Z macro-backtrace for more info) + --> tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.rs:21:12 + | +21 | #[pallet::storage] + | ^^^^^^^ the trait `WrapperTypeEncode` is not implemented for `Bar` + | + = help: the following other types implement trait `WrapperTypeEncode`: + &T + &mut T + Arc + Box + Cow<'a, T> + Rc + Vec + bytes::bytes::Bytes + and $N others + = note: required for `Bar` to implement `Encode` + = note: required for `Bar` to implement `FullEncode` + = note: required for `Bar` to implement `FullCodec` + = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` From 64f464b8b8b5333b68b9b63f4f208e7d3a550f5a Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Thu, 1 Jun 2023 21:51:27 +0530 Subject: [PATCH 45/55] Split items other than storage --- frame/examples/split/src/events.rs | 14 ++++++++ frame/examples/split/src/lib.rs | 20 +++++------ frame/examples/split/src/storages.rs | 12 ------- .../test/tests/split_ui/pass/split_valid.rs | 17 ++++++---- .../pass/split_valid_disambiguation.rs | 34 ++++++++++++------- 5 files changed, 56 insertions(+), 41 deletions(-) create mode 100644 frame/examples/split/src/events.rs delete mode 100644 frame/examples/split/src/storages.rs diff --git a/frame/examples/split/src/events.rs b/frame/examples/split/src/events.rs new file mode 100644 index 0000000000000..fa9d9d8a5c756 --- /dev/null +++ b/frame/examples/split/src/events.rs @@ -0,0 +1,14 @@ +use frame_support::pallet_macros::*; + +#[pallet_section] +mod events { + // Pallets use events to inform users when important changes are made. + // https://docs.substrate.io/main-docs/build/events-errors/ + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + /// Event documentation should end with an array that provides descriptive names for event + /// parameters. [something, who] + SomethingStored { something: u32, who: T::AccountId }, + } +} diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index 59077e956a243..954fbc7f163b2 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -13,12 +13,12 @@ mod tests; #[cfg(feature = "runtime-benchmarks")] mod benchmarking; -mod storages; +mod events; pub mod weights; use frame_support::pallet_macros::*; pub use weights::*; -#[import_section(storages)] +#[import_section(events)] #[frame_support::pallet] pub mod pallet { use super::*; @@ -37,15 +37,13 @@ pub mod pallet { type WeightInfo: WeightInfo; } - // Pallets use events to inform users when important changes are made. - // https://docs.substrate.io/main-docs/build/events-errors/ - #[pallet::event] - #[pallet::generate_deposit(pub(super) fn deposit_event)] - pub enum Event { - /// Event documentation should end with an array that provides descriptive names for event - /// parameters. [something, who] - SomethingStored { something: u32, who: T::AccountId }, - } + // The pallet's runtime storage items. + // https://docs.substrate.io/main-docs/build/runtime-storage/ + #[pallet::storage] + #[pallet::getter(fn something)] + // Learn more about declaring storage items: + // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items + pub type Something = StorageValue<_, u32>; // Errors inform users that something went wrong. #[pallet::error] diff --git a/frame/examples/split/src/storages.rs b/frame/examples/split/src/storages.rs deleted file mode 100644 index e210f6d0b22c6..0000000000000 --- a/frame/examples/split/src/storages.rs +++ /dev/null @@ -1,12 +0,0 @@ -use frame_support::pallet_macros::*; - -#[pallet_section] -mod storages { - // The pallet's runtime storage items. - // https://docs.substrate.io/main-docs/build/runtime-storage/ - #[pallet::storage] - #[pallet::getter(fn something)] - // Learn more about declaring storage items: - // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items - pub type Something = StorageValue<_, u32>; -} diff --git a/frame/support/test/tests/split_ui/pass/split_valid.rs b/frame/support/test/tests/split_ui/pass/split_valid.rs index a1342f49399da..8b5839ecd28a0 100644 --- a/frame/support/test/tests/split_ui/pass/split_valid.rs +++ b/frame/support/test/tests/split_ui/pass/split_valid.rs @@ -5,12 +5,15 @@ use frame_support::pallet_macros::*; pub use pallet::*; #[pallet_section] -mod storages { - #[pallet::storage] - pub type MyStorageMap = StorageMap<_, _, u32, u64>; +mod events { + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + SomethingDone, + } } -#[import_section(storages)] +#[import_section(events)] #[frame_support::pallet(dev_mode)] pub mod pallet { use frame_support::pallet_prelude::*; @@ -20,12 +23,14 @@ pub mod pallet { pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config {} + pub trait Config: frame_system::Config { + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + } #[pallet::call] impl Pallet { pub fn my_call(_origin: OriginFor) -> DispatchResult { - MyStorageMap::::insert(1, 2); + Self::deposit_event(Event::SomethingDone); Ok(()) } } diff --git a/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs b/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs index 7e3c48f11cafe..fea4367c3d52f 100644 --- a/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs +++ b/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs @@ -8,24 +8,29 @@ mod first { use super::*; #[pallet_section] - mod storages { - #[pallet::storage] - pub type MyStorageMap = StorageMap<_, _, u32, u64>; + mod section { + #[pallet::event] + #[pallet::generate_deposit(pub(super) fn deposit_event)] + pub enum Event { + SomethingDone, + } } } mod second { use super::*; - #[pallet_section(storages2)] - mod storages { - #[pallet::storage] - pub type MyStorageMap2 = StorageMap<_, _, u32, u64>; + #[pallet_section(section2)] + mod section { + #[pallet::error] + pub enum Error { + NoneValue, + } } } -#[import_section(storages)] -#[import_section(storages2)] +#[import_section(section)] +#[import_section(section2)] #[frame_support::pallet(dev_mode)] pub mod pallet { use frame_support::pallet_prelude::*; @@ -35,15 +40,20 @@ pub mod pallet { pub struct Pallet(_); #[pallet::config] - pub trait Config: frame_system::Config {} + pub trait Config: frame_system::Config { + type RuntimeEvent: From> + IsType<::RuntimeEvent>; + } #[pallet::call] impl Pallet { pub fn my_call(_origin: OriginFor) -> DispatchResult { - MyStorageMap::::insert(1, 2); - MyStorageMap2::::insert(1, 2); + Self::deposit_event(Event::SomethingDone); Ok(()) } + + pub fn my_call_2(_origin: OriginFor) -> DispatchResult { + return Err(Error::::NoneValue.into()) + } } } From f3cd00520d9d522d1f574a310e98b8f72950ec7e Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Fri, 2 Jun 2023 10:17:21 +0530 Subject: [PATCH 46/55] Updates versions --- frame/examples/split/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/examples/split/Cargo.toml b/frame/examples/split/Cargo.toml index 84e470b7a4457..c7221c3bbb2c8 100644 --- a/frame/examples/split/Cargo.toml +++ b/frame/examples/split/Cargo.toml @@ -20,13 +20,13 @@ frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" } -sp-io = { version = "8.0.0", default-features = false, path = "../../../primitives/io" } -sp-runtime = { version = "8.0.0", default-features = false, path = "../../../primitives/runtime" } -sp-std = { version = "6.0.0", default-features = false, path = "../../../primitives/std" } +sp-io = { version = "23.0.0", default-features = false, path = "../../../primitives/io" } +sp-runtime = { version = "24.0.0", default-features = false, path = "../../../primitives/runtime" } +sp-std = { version = "8.0.0", default-features = false, path = "../../../primitives/std" } macro_magic = "0.3.1" [dev-dependencies] -sp-core = { version = "8.0.0", default-features = false, path = "../../../primitives/core" } +sp-core = { version = "21.0.0", default-features = false, path = "../../../primitives/core" } [features] default = ["std"] From 0c444ceb96a4a3c3f75f13c74574ec8394c3b809 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 21 Jun 2023 11:52:05 +0530 Subject: [PATCH 47/55] Fixes some review comments --- frame/examples/split/Cargo.toml | 25 +++++++++++++++------- frame/examples/split/src/benchmarking.rs | 21 +++++++++++++++++- frame/examples/split/src/events.rs | 19 +++++++++++++++-- frame/examples/split/src/lib.rs | 17 +++++++++++++++ frame/examples/split/src/mock.rs | 27 ++++++------------------ 5 files changed, 78 insertions(+), 31 deletions(-) diff --git a/frame/examples/split/Cargo.toml b/frame/examples/split/Cargo.toml index c7221c3bbb2c8..e6f61a99764d3 100644 --- a/frame/examples/split/Cargo.toml +++ b/frame/examples/split/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" license = "MIT-0" homepage = "https://substrate.io" repository = "https://github.com/paritytech/substrate/" -description = "FRAME example pallet" +description = "FRAME example splitted pallet" readme = "README.md" [package.metadata.docs.rs] @@ -16,14 +16,19 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.2.2", default-features = false } log = { version = "0.4.17", default-features = false } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } -frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../benchmarking" } + frame-support = { version = "4.0.0-dev", default-features = false, path = "../../support" } frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } -pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" } + sp-io = { version = "23.0.0", default-features = false, path = "../../../primitives/io" } sp-runtime = { version = "24.0.0", default-features = false, path = "../../../primitives/runtime" } sp-std = { version = "8.0.0", default-features = false, path = "../../../primitives/std" } -macro_magic = "0.3.1" + +frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../benchmarking" } + +pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" } + +macro_magic = "0.4.1" [dev-dependencies] sp-core = { version = "21.0.0", default-features = false, path = "../../../primitives/core" } @@ -32,15 +37,19 @@ sp-core = { version = "21.0.0", default-features = false, path = "../../../primi default = ["std"] std = [ "codec/std", - "frame-benchmarking?/std", - "frame-support/std", - "frame-system/std", "log/std", - "pallet-balances/std", "scale-info/std", + + "frame-support/std", + "frame-system/std", + "sp-io/std", "sp-runtime/std", "sp-std/std", + + "frame-benchmarking?/std", + + "pallet-balances/std", ] runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/examples/split/src/benchmarking.rs b/frame/examples/split/src/benchmarking.rs index 5a262417629c5..5a50300937203 100644 --- a/frame/examples/split/src/benchmarking.rs +++ b/frame/examples/split/src/benchmarking.rs @@ -1,4 +1,23 @@ -//! Benchmarking setup for pallet-template +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Benchmarking setup for pallet-example-split + +// Only enable this module for benchmarking. #![cfg(feature = "runtime-benchmarks")] use super::*; diff --git a/frame/examples/split/src/events.rs b/frame/examples/split/src/events.rs index fa9d9d8a5c756..1b06f37b3203e 100644 --- a/frame/examples/split/src/events.rs +++ b/frame/examples/split/src/events.rs @@ -1,9 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use frame_support::pallet_macros::*; #[pallet_section] mod events { - // Pallets use events to inform users when important changes are made. - // https://docs.substrate.io/main-docs/build/events-errors/ #[pallet::event] #[pallet::generate_deposit(pub(super) fn deposit_event)] pub enum Event { diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index 954fbc7f163b2..58ce9555574ee 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #![cfg_attr(not(feature = "std"), no_std)] /// Edit this file to define custom logic or remove it if it is not needed. diff --git a/frame/examples/split/src/mock.rs b/frame/examples/split/src/mock.rs index b4d6905378a5d..bc7c2626c47c3 100644 --- a/frame/examples/split/src/mock.rs +++ b/frame/examples/split/src/mock.rs @@ -1,5 +1,5 @@ use crate as pallet_template; -use frame_support::traits::{ConstU16, ConstU64}; +use frame_support::{derive_impl, traits::{ConstU16, ConstU64}}; use sp_core::H256; use sp_runtime::{ testing::Header, @@ -17,35 +17,22 @@ frame_support::construct_runtime!( UncheckedExtrinsic = UncheckedExtrinsic, { System: frame_system, - TemplateModule: pallet_template, + TemplatePallet: pallet_template, } ); +/// Using a default config for [`frame_system`] in tests. See `default-config` example for more +/// details. +#[derive_impl(frame_system::config_preludes::TestDefaultConfig as frame_system::DefaultConfig)] impl frame_system::Config for Test { type BaseCallFilter = frame_support::traits::Everything; - type BlockWeights = (); - type BlockLength = (); - type DbWeight = (); type RuntimeOrigin = RuntimeOrigin; type RuntimeCall = RuntimeCall; - type Index = u64; - type BlockNumber = u64; - type Hash = H256; - type Hashing = BlakeTwo256; - type AccountId = u64; - type Lookup = IdentityLookup; - type Header = Header; type RuntimeEvent = RuntimeEvent; - type BlockHashCount = ConstU64<250>; - type Version = (); type PalletInfo = PalletInfo; - type AccountData = (); - type OnNewAccount = (); - type OnKilledAccount = (); - type SystemWeightInfo = (); - type SS58Prefix = ConstU16<42>; type OnSetCode = (); - type MaxConsumers = frame_support::traits::ConstU32<16>; + + type AccountData = pallet_balances::AccountData; } impl pallet_template::Config for Test { From 52193708840b061548a1ccb9cf9211f276d09733 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 21 Jun 2023 18:33:56 +0530 Subject: [PATCH 48/55] Addresses review comments --- Cargo.lock | 19 +++++++++++++++++-- frame/examples/Cargo.toml | 5 ++++- frame/examples/split/Cargo.toml | 6 ------ frame/examples/split/src/lib.rs | 2 +- frame/examples/split/src/mock.rs | 9 +-------- frame/examples/split/src/tests.rs | 6 +++--- frame/examples/split/src/weights.rs | 16 ++++++++-------- frame/examples/src/lib.rs | 3 +++ frame/support/procedural/src/lib.rs | 13 ++++++++++++- frame/support/src/lib.rs | 1 - frame/support/test/Cargo.toml | 1 - .../split_ui/import_without_pallet.stderr | 16 +++------------- .../tests/split_ui/no_section_found.stderr | 2 +- .../pass/split_valid_disambiguation.rs | 4 ++-- 14 files changed, 55 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 68670700ec8fc..c0982a4af4a11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2887,7 +2887,6 @@ dependencies = [ "frame-support", "frame-support-test-pallet", "frame-system", - "macro_magic", "parity-scale-codec", "pretty_assertions", "rustversion", @@ -6588,6 +6587,22 @@ dependencies = [ "sp-std", ] +[[package]] +name = "pallet-example-split" +version = "4.0.0-dev" +dependencies = [ + "frame-benchmarking", + "frame-support", + "frame-system", + "log", + "parity-scale-codec", + "scale-info", + "sp-core", + "sp-io", + "sp-runtime", + "sp-std", +] + [[package]] name = "pallet-examples" version = "4.0.0-dev" @@ -6597,7 +6612,7 @@ dependencies = [ "pallet-example-basic", "pallet-example-kitchensink", "pallet-example-offchain-worker", - "pallet-example-split" + "pallet-example-split", ] [[package]] diff --git a/frame/examples/Cargo.toml b/frame/examples/Cargo.toml index 57def091b1f63..af67bef792b6f 100644 --- a/frame/examples/Cargo.toml +++ b/frame/examples/Cargo.toml @@ -17,6 +17,7 @@ pallet-default-config-example = { default-features = false, path = "./default-co pallet-example-offchain-worker = { default-features = false, path = "./offchain-worker" } pallet-example-kitchensink = { default-features = false, path = "./kitchensink" } pallet-dev-mode = { default-features = false, path = "./dev-mode" } +pallet-example-split = { default-features = false, path = "./split" } [features] default = [ "std" ] @@ -26,6 +27,7 @@ std = [ "pallet-example-offchain-worker/std", "pallet-example-kitchensink/std", "pallet-dev-mode/std", + "pallet-example-split/std", ] try-runtime = [ "pallet-example-basic/try-runtime", @@ -33,4 +35,5 @@ try-runtime = [ "pallet-example-offchain-worker/try-runtime", "pallet-example-kitchensink/try-runtime", "pallet-dev-mode/try-runtime", -] \ No newline at end of file + "pallet-example-split/try-runtime", +] diff --git a/frame/examples/split/Cargo.toml b/frame/examples/split/Cargo.toml index e6f61a99764d3..08ba63fdd51f6 100644 --- a/frame/examples/split/Cargo.toml +++ b/frame/examples/split/Cargo.toml @@ -26,10 +26,6 @@ sp-std = { version = "8.0.0", default-features = false, path = "../../../primiti frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../benchmarking" } -pallet-balances = { version = "4.0.0-dev", default-features = false, path = "../../balances" } - -macro_magic = "0.4.1" - [dev-dependencies] sp-core = { version = "21.0.0", default-features = false, path = "../../../primitives/core" } @@ -48,8 +44,6 @@ std = [ "sp-std/std", "frame-benchmarking?/std", - - "pallet-balances/std", ] runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks"] try-runtime = ["frame-support/try-runtime"] diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index 58ce9555574ee..b0b065c983119 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -35,7 +35,7 @@ pub mod weights; use frame_support::pallet_macros::*; pub use weights::*; -#[import_section(events)] +#[import_section(events::events)] #[frame_support::pallet] pub mod pallet { use super::*; diff --git a/frame/examples/split/src/mock.rs b/frame/examples/split/src/mock.rs index bc7c2626c47c3..f373f80a87913 100644 --- a/frame/examples/split/src/mock.rs +++ b/frame/examples/split/src/mock.rs @@ -1,10 +1,5 @@ use crate as pallet_template; -use frame_support::{derive_impl, traits::{ConstU16, ConstU64}}; -use sp_core::H256; -use sp_runtime::{ - testing::Header, - traits::{BlakeTwo256, IdentityLookup}, -}; +use frame_support::derive_impl; type UncheckedExtrinsic = frame_system::mocking::MockUncheckedExtrinsic; type Block = frame_system::mocking::MockBlock; @@ -31,8 +26,6 @@ impl frame_system::Config for Test { type RuntimeEvent = RuntimeEvent; type PalletInfo = PalletInfo; type OnSetCode = (); - - type AccountData = pallet_balances::AccountData; } impl pallet_template::Config for Test { diff --git a/frame/examples/split/src/tests.rs b/frame/examples/split/src/tests.rs index 7c2b853ee4dc5..e76ea20778a7c 100644 --- a/frame/examples/split/src/tests.rs +++ b/frame/examples/split/src/tests.rs @@ -7,9 +7,9 @@ fn it_works_for_default_value() { // Go past genesis block so events get deposited System::set_block_number(1); // Dispatch a signed extrinsic. - assert_ok!(TemplateModule::do_something(RuntimeOrigin::signed(1), 42)); + assert_ok!(TemplatePallet::do_something(RuntimeOrigin::signed(1), 42)); // Read pallet storage and assert an expected result. - assert_eq!(TemplateModule::something(), Some(42)); + assert_eq!(TemplatePallet::something(), Some(42)); // Assert that the correct event was deposited System::assert_last_event(Event::SomethingStored { something: 42, who: 1 }.into()); }); @@ -20,7 +20,7 @@ fn correct_error_for_none_value() { new_test_ext().execute_with(|| { // Ensure the expected error is thrown when no value is present. assert_noop!( - TemplateModule::cause_error(RuntimeOrigin::signed(1)), + TemplatePallet::cause_error(RuntimeOrigin::signed(1)), Error::::NoneValue ); }); diff --git a/frame/examples/split/src/weights.rs b/frame/examples/split/src/weights.rs index e8fbc09bad8e9..4219ce1e2697c 100644 --- a/frame/examples/split/src/weights.rs +++ b/frame/examples/split/src/weights.rs @@ -42,8 +42,8 @@ pub trait WeightInfo { /// Weights for pallet_template using the Substrate node and recommended hardware. pub struct SubstrateWeight(PhantomData); impl WeightInfo for SubstrateWeight { - /// Storage: TemplateModule Something (r:0 w:1) - /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + /// Storage: TemplatePallet Something (r:0 w:1) + /// Proof: TemplatePallet Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) fn do_something() -> Weight { // Proof Size summary in bytes: // Measured: `0` @@ -52,8 +52,8 @@ impl WeightInfo for SubstrateWeight { Weight::from_parts(9_000_000, 0) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: TemplateModule Something (r:1 w:1) - /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + /// Storage: TemplatePallet Something (r:1 w:1) + /// Proof: TemplatePallet Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) fn cause_error() -> Weight { // Proof Size summary in bytes: // Measured: `32` @@ -67,8 +67,8 @@ impl WeightInfo for SubstrateWeight { // For backwards compatibility and tests impl WeightInfo for () { - /// Storage: TemplateModule Something (r:0 w:1) - /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + /// Storage: TemplatePallet Something (r:0 w:1) + /// Proof: TemplatePallet Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) fn do_something() -> Weight { // Proof Size summary in bytes: // Measured: `0` @@ -77,8 +77,8 @@ impl WeightInfo for () { Weight::from_parts(9_000_000, 0) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: TemplateModule Something (r:1 w:1) - /// Proof: TemplateModule Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) + /// Storage: TemplatePallet Something (r:1 w:1) + /// Proof: TemplatePallet Something (max_values: Some(1), max_size: Some(4), added: 499, mode: MaxEncodedLen) fn cause_error() -> Weight { // Proof Size summary in bytes: // Measured: `32` diff --git a/frame/examples/src/lib.rs b/frame/examples/src/lib.rs index 3ff1aa7b9f1f2..b5ca2f57bd542 100644 --- a/frame/examples/src/lib.rs +++ b/frame/examples/src/lib.rs @@ -35,3 +35,6 @@ //! //! - [**`pallet-example-kitchensink`**](./kitchensink): A simple example of a FRAME pallet //! demonstrating a catalog of the the FRAME macros and their various syntax options. +//! +//! - [**`pallet-example-split`**](./split): A simple example of a FRAME pallet +//! demonstrating the ability to split sections across multiple files. diff --git a/frame/support/procedural/src/lib.rs b/frame/support/procedural/src/lib.rs index 105cb40cb10b6..1f4162a0b7551 100644 --- a/frame/support/procedural/src/lib.rs +++ b/frame/support/procedural/src/lib.rs @@ -1827,7 +1827,18 @@ pub fn pallet_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { /// /// Note that sections are imported by their module name/ident, and should be referred to by /// their _full path_ from the perspective of the target pallet. -#[import_tokens_attr] +#[import_tokens_attr { + format!( + "{}::macro_magic", + match generate_crate_access_2018("frame-support") { + Ok(path) => Ok(path), + Err(_) => generate_crate_access_2018("frame"), + } + .expect("Failed to find either `frame-support` or `frame` in `Cargo.toml` dependencies.") + .to_token_stream() + .to_string() + ) +}] #[proc_macro_attribute] pub fn import_section(attr: TokenStream, tokens: TokenStream) -> TokenStream { let foreign_mod = parse_macro_input!(attr as ItemMod); diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index eb522ac00b585..816f9536ea5d0 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2896,7 +2896,6 @@ pub use frame_support_procedural::pallet; /// Contains macro stubs for all of the pallet:: macros pub mod pallet_macros { - #[macro_magic::use_attr] pub use frame_support_procedural::import_section; pub use frame_support_procedural::{ call_index, compact, composite_enum, config, constant, diff --git a/frame/support/test/Cargo.toml b/frame/support/test/Cargo.toml index 60ad5a44c574c..555d84f5072bd 100644 --- a/frame/support/test/Cargo.toml +++ b/frame/support/test/Cargo.toml @@ -33,7 +33,6 @@ frame-system = { version = "4.0.0-dev", default-features = false, path = "../../ frame-executive = { version = "4.0.0-dev", default-features = false, path = "../../executive" } # The "std" feature for this pallet is never activated on purpose, in order to test construct_runtime error message test-pallet = { package = "frame-support-test-pallet", default-features = false, path = "pallet" } -macro_magic = "0.3.1" [features] default = ["std"] diff --git a/frame/support/test/tests/split_ui/import_without_pallet.stderr b/frame/support/test/tests/split_ui/import_without_pallet.stderr index 4a1b8d44fe88f..0d7b5414b1016 100644 --- a/frame/support/test/tests/split_ui/import_without_pallet.stderr +++ b/frame/support/test/tests/split_ui/import_without_pallet.stderr @@ -1,15 +1,5 @@ error: `#[import_section]` can only be applied to a valid pallet module - --> tests/split_ui/import_without_pallet.rs:5:1 + --> tests/split_ui/import_without_pallet.rs:12:9 | -5 | / #[pallet_section] -6 | | mod storages { -7 | | #[pallet::storage] -8 | | pub type MyStorageMap = StorageMap<_, _, u32, u64>; -9 | | } -10 | | -11 | | #[import_section(storages)] - | |_--------------------------^ - | | - | in this procedural macro expansion - | - = note: this error originates in the macro `__import_tokens_attr_import_section_inner` which comes from the expansion of the attribute macro `import_section` (in Nightly builds, run with -Z macro-backtrace for more info) +12 | pub mod pallet { + | ^^^^^^ diff --git a/frame/support/test/tests/split_ui/no_section_found.stderr b/frame/support/test/tests/split_ui/no_section_found.stderr index 05c38c904f97d..e0a9322b188e3 100644 --- a/frame/support/test/tests/split_ui/no_section_found.stderr +++ b/frame/support/test/tests/split_ui/no_section_found.stderr @@ -10,4 +10,4 @@ error: cannot find macro `__export_tokens_tt_storages_dev` in this scope 7 | #[import_section(storages_dev)] | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ | - = note: this error originates in the macro `::macro_magic::forward_tokens` (in Nightly builds, run with -Z macro-backtrace for more info) + = note: this error originates in the macro `frame_support::macro_magic::forward_tokens` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs b/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs index fea4367c3d52f..8d8d50422e9ce 100644 --- a/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs +++ b/frame/support/test/tests/split_ui/pass/split_valid_disambiguation.rs @@ -29,8 +29,8 @@ mod second { } } -#[import_section(section)] -#[import_section(section2)] +#[import_section(first::section)] +#[import_section(second::section2)] #[frame_support::pallet(dev_mode)] pub mod pallet { use frame_support::pallet_prelude::*; From 72d4204e0aa2be0763953c99361938d9ceb88d40 Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Wed, 21 Jun 2023 13:21:46 +0000 Subject: [PATCH 49/55] ".git/.scripts/commands/fmt/fmt.sh" --- frame/examples/src/lib.rs | 6 +++--- frame/support/src/lib.rs | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/frame/examples/src/lib.rs b/frame/examples/src/lib.rs index b5ca2f57bd542..d1cd32bb50f26 100644 --- a/frame/examples/src/lib.rs +++ b/frame/examples/src/lib.rs @@ -35,6 +35,6 @@ //! //! - [**`pallet-example-kitchensink`**](./kitchensink): A simple example of a FRAME pallet //! demonstrating a catalog of the the FRAME macros and their various syntax options. -//! -//! - [**`pallet-example-split`**](./split): A simple example of a FRAME pallet -//! demonstrating the ability to split sections across multiple files. +//! +//! - [**`pallet-example-split`**](./split): A simple example of a FRAME pallet demonstrating the +//! ability to split sections across multiple files. diff --git a/frame/support/src/lib.rs b/frame/support/src/lib.rs index 816f9536ea5d0..6b020716df54c 100644 --- a/frame/support/src/lib.rs +++ b/frame/support/src/lib.rs @@ -2896,13 +2896,12 @@ pub use frame_support_procedural::pallet; /// Contains macro stubs for all of the pallet:: macros pub mod pallet_macros { - pub use frame_support_procedural::import_section; pub use frame_support_procedural::{ call_index, compact, composite_enum, config, constant, disable_frame_system_supertrait_check, error, event, extra_constants, generate_deposit, - generate_store, genesis_build, genesis_config, getter, hooks, inherent, no_default, origin, - pallet_section, storage, storage_prefix, storage_version, type_value, unbounded, - validate_unsigned, weight, whitelist_storage, + generate_store, genesis_build, genesis_config, getter, hooks, import_section, inherent, + no_default, origin, pallet_section, storage, storage_prefix, storage_version, type_value, + unbounded, validate_unsigned, weight, whitelist_storage, }; } From 17b9cdb3da30f001d7e67894e51c27be5b9c78c7 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 21 Jun 2023 19:09:12 +0530 Subject: [PATCH 50/55] Updates docs --- Cargo.lock | 1 - frame/examples/split/Cargo.toml | 2 -- frame/examples/split/src/events.rs | 2 ++ frame/examples/split/src/lib.rs | 18 +++++++++++++----- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0982a4af4a11..14591ed0270b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6599,7 +6599,6 @@ dependencies = [ "scale-info", "sp-core", "sp-io", - "sp-runtime", "sp-std", ] diff --git a/frame/examples/split/Cargo.toml b/frame/examples/split/Cargo.toml index 08ba63fdd51f6..21ecf89330dd7 100644 --- a/frame/examples/split/Cargo.toml +++ b/frame/examples/split/Cargo.toml @@ -21,7 +21,6 @@ frame-support = { version = "4.0.0-dev", default-features = false, path = "../.. frame-system = { version = "4.0.0-dev", default-features = false, path = "../../system" } sp-io = { version = "23.0.0", default-features = false, path = "../../../primitives/io" } -sp-runtime = { version = "24.0.0", default-features = false, path = "../../../primitives/runtime" } sp-std = { version = "8.0.0", default-features = false, path = "../../../primitives/std" } frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../../benchmarking" } @@ -40,7 +39,6 @@ std = [ "frame-system/std", "sp-io/std", - "sp-runtime/std", "sp-std/std", "frame-benchmarking?/std", diff --git a/frame/examples/split/src/events.rs b/frame/examples/split/src/events.rs index 1b06f37b3203e..7e0f9e15fd6f1 100644 --- a/frame/examples/split/src/events.rs +++ b/frame/examples/split/src/events.rs @@ -17,6 +17,8 @@ use frame_support::pallet_macros::*; +/// A [`pallet_section`] that defines the events for a pallet. +/// This can later be imported into the pallet using [`import_section`]. #[pallet_section] mod events { #[pallet::event] diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index b0b065c983119..f7efa1f8298c3 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -15,11 +15,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +//! # Split Example Pallet +//! +//! **This pallet serves as an example and is not meant to be used in production.** +//! +//! A FRAME pallet demonstrating the ability to split sections across multiple files. + #![cfg_attr(not(feature = "std"), no_std)] -/// Edit this file to define custom logic or remove it if it is not needed. -/// Learn more about FRAME and the core library of Substrate FRAME pallets: -/// +// Re-export pallet items so that they can be accessed from the crate namespace. pub use pallet::*; #[cfg(test)] @@ -31,10 +35,14 @@ mod tests; #[cfg(feature = "runtime-benchmarks")] mod benchmarking; mod events; + pub mod weights; -use frame_support::pallet_macros::*; pub use weights::*; +use frame_support::pallet_macros::*; + +/// Imports a [`pallet_section`] defined at [`events::events`]. +/// This brings the events defined in that section into the pallet's namespace. #[import_section(events::events)] #[frame_support::pallet] pub mod pallet { @@ -57,7 +65,7 @@ pub mod pallet { // The pallet's runtime storage items. // https://docs.substrate.io/main-docs/build/runtime-storage/ #[pallet::storage] - #[pallet::getter(fn something)] + #[pallet::getter(fn something)] // optional // Learn more about declaring storage items: // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items pub type Something = StorageValue<_, u32>; From 125b5dca28d93a427ea8fdeeb9c95c5b576eb36e Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Wed, 21 Jun 2023 19:11:33 +0530 Subject: [PATCH 51/55] Adds experimental disclaimer --- frame/examples/split/README.md | 5 ++++- frame/examples/split/src/lib.rs | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/frame/examples/split/README.md b/frame/examples/split/README.md index ddf8d2a9b1c8f..413ce9b913cb9 100644 --- a/frame/examples/split/README.md +++ b/frame/examples/split/README.md @@ -1,6 +1,9 @@ # Basic Example For Splitting A Pallet -A simple example of a FRAME pallet demonstrating the ability to split sections across multiple files. +A simple example of a FRAME pallet demonstrating the ability to split sections across multiple +files. + +Note that this is purely experimental at this point. Run `cargo doc --package pallet-example-split --open` to view this pallet's documentation. diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index f7efa1f8298c3..40a30cfcb6eb2 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -20,6 +20,8 @@ //! **This pallet serves as an example and is not meant to be used in production.** //! //! A FRAME pallet demonstrating the ability to split sections across multiple files. +//! +//! Note that this is purely experimental at this point. #![cfg_attr(not(feature = "std"), no_std)] From 3fa8d5caa5fccfe5a7e5ffb22ad884b616dcb127 Mon Sep 17 00:00:00 2001 From: command-bot <> Date: Wed, 21 Jun 2023 13:45:29 +0000 Subject: [PATCH 52/55] ".git/.scripts/commands/fmt/fmt.sh" --- frame/examples/split/src/events.rs | 2 +- frame/examples/split/src/lib.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frame/examples/split/src/events.rs b/frame/examples/split/src/events.rs index 7e0f9e15fd6f1..7560766bacb33 100644 --- a/frame/examples/split/src/events.rs +++ b/frame/examples/split/src/events.rs @@ -17,7 +17,7 @@ use frame_support::pallet_macros::*; -/// A [`pallet_section`] that defines the events for a pallet. +/// A [`pallet_section`] that defines the events for a pallet. /// This can later be imported into the pallet using [`import_section`]. #[pallet_section] mod events { diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index 40a30cfcb6eb2..1cfce1ddbe519 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -20,7 +20,7 @@ //! **This pallet serves as an example and is not meant to be used in production.** //! //! A FRAME pallet demonstrating the ability to split sections across multiple files. -//! +//! //! Note that this is purely experimental at this point. #![cfg_attr(not(feature = "std"), no_std)] @@ -68,8 +68,8 @@ pub mod pallet { // https://docs.substrate.io/main-docs/build/runtime-storage/ #[pallet::storage] #[pallet::getter(fn something)] // optional - // Learn more about declaring storage items: - // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items + // Learn more about declaring storage items: + // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items pub type Something = StorageValue<_, u32>; // Errors inform users that something went wrong. From ebf806df91b4bd9fcd613e5d0bb9ee534a19602f Mon Sep 17 00:00:00 2001 From: gupnik <17176722+gupnik@users.noreply.github.com> Date: Wed, 21 Jun 2023 21:37:30 +0530 Subject: [PATCH 53/55] Update frame/support/test/tests/split_ui/no_section_found.rs Co-authored-by: Sam Johnson --- frame/support/test/tests/split_ui/no_section_found.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame/support/test/tests/split_ui/no_section_found.rs b/frame/support/test/tests/split_ui/no_section_found.rs index b44a7e9015206..fe12c6dc51b72 100644 --- a/frame/support/test/tests/split_ui/no_section_found.rs +++ b/frame/support/test/tests/split_ui/no_section_found.rs @@ -26,4 +26,4 @@ pub mod pallet { } fn main() { -} \ No newline at end of file +} From 04aefd2a826bf170eca195bbbc9ddeed20b532f1 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 27 Jun 2023 16:27:39 +0530 Subject: [PATCH 54/55] Addresses review comments --- frame/examples/split/src/lib.rs | 5 ----- frame/examples/split/src/mock.rs | 17 +++++++++++++++++ frame/examples/split/src/tests.rs | 17 +++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/frame/examples/split/src/lib.rs b/frame/examples/split/src/lib.rs index 40a30cfcb6eb2..df994b5bbac8a 100644 --- a/frame/examples/split/src/lib.rs +++ b/frame/examples/split/src/lib.rs @@ -65,11 +65,7 @@ pub mod pallet { } // The pallet's runtime storage items. - // https://docs.substrate.io/main-docs/build/runtime-storage/ #[pallet::storage] - #[pallet::getter(fn something)] // optional - // Learn more about declaring storage items: - // https://docs.substrate.io/main-docs/build/runtime-storage/#declaring-storage-items pub type Something = StorageValue<_, u32>; // Errors inform users that something went wrong. @@ -93,7 +89,6 @@ pub mod pallet { pub fn do_something(origin: OriginFor, something: u32) -> DispatchResult { // Check that the extrinsic was signed and get the signer. // This function will return an error if the extrinsic is not signed. - // https://docs.substrate.io/main-docs/build/origins/ let who = ensure_signed(origin)?; // Update storage. diff --git a/frame/examples/split/src/mock.rs b/frame/examples/split/src/mock.rs index f373f80a87913..e479e821f4262 100644 --- a/frame/examples/split/src/mock.rs +++ b/frame/examples/split/src/mock.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use crate as pallet_template; use frame_support::derive_impl; diff --git a/frame/examples/split/src/tests.rs b/frame/examples/split/src/tests.rs index e76ea20778a7c..d070aeb1ba2bb 100644 --- a/frame/examples/split/src/tests.rs +++ b/frame/examples/split/src/tests.rs @@ -1,3 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + use crate::{mock::*, Error, Event}; use frame_support::{assert_noop, assert_ok}; From dd96d1a5ecdfc5b30b93ed85871825fd208b1344 Mon Sep 17 00:00:00 2001 From: Nikhil Gupta <17176722+gupnik@users.noreply.github.com> Date: Tue, 27 Jun 2023 16:50:21 +0530 Subject: [PATCH 55/55] Fixes test --- frame/examples/split/src/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame/examples/split/src/tests.rs b/frame/examples/split/src/tests.rs index d070aeb1ba2bb..1d4b6dfcff9d5 100644 --- a/frame/examples/split/src/tests.rs +++ b/frame/examples/split/src/tests.rs @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use crate::{mock::*, Error, Event}; +use crate::{mock::*, Error, Event, Something}; use frame_support::{assert_noop, assert_ok}; #[test] @@ -26,7 +26,7 @@ fn it_works_for_default_value() { // Dispatch a signed extrinsic. assert_ok!(TemplatePallet::do_something(RuntimeOrigin::signed(1), 42)); // Read pallet storage and assert an expected result. - assert_eq!(TemplatePallet::something(), Some(42)); + assert_eq!(Something::::get(), Some(42)); // Assert that the correct event was deposited System::assert_last_event(Event::SomethingStored { something: 42, who: 1 }.into()); });