Introduce Pallet paged-list

Cargo.toml

@@ -138,6 +138,8 @@ members = [
 	"frame/nomination-pools/benchmarking",
 	"frame/nomination-pools/test-staking",
 	"frame/nomination-pools/runtime-api",
+	"frame/paged-list",
+	"frame/paged-list/fuzzer",
 	"frame/insecure-randomness-collective-flip",
 	"frame/ranked-collective",
 	"frame/recovery",

frame/paged-list/Cargo.toml

@@ -0,0 +1,35 @@
+[package]
+name = "pallet-paged-list"
+version = "0.1.0"
+description = "FRAME pallet that provides a paged list data structure."
+authors = ["Parity Technologies <admin@parity.io>"]
+homepage = "https://substrate.io"
+edition = "2021"
+license = "Apache-2.0"
+repository = "https://github.com/paritytech/substrate"
+
+[package.metadata.docs.rs]
+targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ "derive"] }
+docify = "0.1.13"
+scale-info = { version = "2.0.0", default-features = false, features = ["derive"] }
+
+frame-benchmarking = { version = "4.0.0-dev", default-features = false, optional = true, path = "../benchmarking" }
+frame-support = { version = "4.0.0-dev", default-features = false, path = "../support" }
+frame-system = { version = "4.0.0-dev", default-features = false, path = "../system" }
+
+sp-runtime = { version = "24.0.0", default-features = false, path = "../../primitives/runtime" }
+sp-std = { version = "8.0.0", default-features = false, path = "../../primitives/std" }
+sp-core = { version = "21.0.0", path = "../../primitives/core", default-features = false }
+sp-io = { version = "23.0.0", path = "../../primitives/io", default-features = false }
+
+[features]
+default = ["std"]
+
+std = ["codec/std", "frame-benchmarking?/std", "frame-support/std", "frame-system/std", "scale-info/std", "sp-core/std", "sp-io/std", "sp-runtime/std", "sp-std/std"]
+
+runtime-benchmarks = ["frame-benchmarking/runtime-benchmarks", "frame-support/runtime-benchmarks", "frame-system/runtime-benchmarks", "sp-runtime/runtime-benchmarks"]
+
+try-runtime = ["frame-support/try-runtime"]

frame/paged-list/fuzzer/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "pallet-paged-list-fuzzer"
+version = "0.1.0"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2021"
+license = "Apache-2.0"
+homepage = "https://substrate.io"
+repository = "https://github.com/paritytech/substrate/"
+description = "Fuzz storage types of pallet-paged-list"
+publish = false
+
+[[bin]]
+name = "pallet-paged-list"
+path = "src/paged_list.rs"
+
+[dependencies]
+arbitrary = "1.3.0"
+honggfuzz = "0.5.49"
+
+frame-support = { version = "4.0.0-dev", default-features = false, features = [ "std" ], path = "../../support" }
+sp-io = { path = "../../../primitives/io", default-features = false, features = [ "std" ] }
+pallet-paged-list = { path = "../", default-features = false, features = [ "std" ] }

frame/paged-list/fuzzer/src/paged_list.rs

@@ -0,0 +1,103 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! # Running
+//! Running this fuzzer can be done with `cargo hfuzz run pallet-paged-list`. `honggfuzz` CLI
+//! options can be used by setting `HFUZZ_RUN_ARGS`, such as `-n 4` to use 4 threads.
+//!
+//! # Debugging a panic
+//! Once a panic is found, it can be debugged with
+//! `cargo hfuzz run-debug pallet-paged-list hfuzz_workspace/pallet-paged-list/*.fuzz`.
+//!
+//! # More information
+//! More information about `honggfuzz` can be found
+//! [here](https://docs.rs/honggfuzz/).
+
+use arbitrary::Arbitrary;
+use honggfuzz::fuzz;
+
+use frame_support::{storage::StorageList, StorageNoopGuard};
+use pallet_paged_list::mock::{PagedList as List, *};
+use sp_io::TestExternalities;
+type Meta = MetaOf<Test, ()>;
+
+fn main() {
+	loop {
+		fuzz!(|data: (Vec<Op>, u8)| {
+			drain_append_work(data.0, data.1);
+		});
+	}
+}
+
+/// Appends and drains random number of elements in random order and checks storage invariants.
+///
+/// It also changes the maximal number of elements per page dynamically, hence the `page_size`.
+fn drain_append_work(ops: Vec<Op>, page_size: u8) {
+	if page_size == 0 {
+		return
+	}
+
+	TestExternalities::default().execute_with(|| {
+		ValuesPerNewPage::set(&page_size.into());
+		let _g = StorageNoopGuard::default();
+		let mut total: i64 = 0;
+
+		for op in ops.into_iter() {
+			total += op.exec();
+
+			assert!(total >= 0);
+			assert_eq!(List::iter().count(), total as usize);
+
+			// We have the assumption that the queue removes the metadata when empty.
+			if total == 0 {
+				assert_eq!(List::drain().count(), 0);
+				assert_eq!(Meta::from_storage().unwrap_or_default(), Default::default());
+			}
+		}
+
+		assert_eq!(List::drain().count(), total as usize);
+		// `StorageNoopGuard` checks that there is no storage leaked.
+	});
+}
+
+enum Op {
+	Append(Vec<u32>),
+	Drain(u8),
+}
+
+impl Arbitrary<'_> for Op {
+	fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> {
+		if u.arbitrary::<bool>()? {
+			Ok(Op::Append(Vec::<u32>::arbitrary(u)?))
+		} else {
+			Ok(Op::Drain(u.arbitrary::<u8>()?))
+		}
+	}
+}
+
+impl Op {
+	pub fn exec(self) -> i64 {
+		match self {
+			Op::Append(v) => {
+				let l = v.len();
+				List::append_many(v);
+				l as i64
+			},
+			Op::Drain(v) => -(List::drain().take(v as usize).count() as i64),
+		}
+	}
+}

frame/paged-list/src/lib.rs

@@ -0,0 +1,136 @@
+// This file is part of Substrate.
+
+// Copyright (C) Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! > Made with *Substrate*, for *DotSama*.
+//!
+//! [![github]](https://github.com/paritytech/substrate/frame/fast-unstake) -
+//! [![polkadot]](https://polkadot.network)
+//!
+//! [polkadot]: https://img.shields.io/badge/polkadot-E6007A?style=for-the-badge&logo=polkadot&logoColor=white
+//! [github]: https://img.shields.io/badge/github-8da0cb?style=for-the-badge&labelColor=555555&logo=github
+//!
+//! # Paged List Pallet
+//!
+//! A thin wrapper pallet around a [`paged_list::StoragePagedList`]. It provides an API for a single
+//! paginated list. It can be instantiated multiple times to provide multiple lists.
+//!
+//! ## Overview
+//!
+//! The pallet is quite unique since it does not expose any `Call`s, `Error`s or `Event`s. All
+//! interaction goes through the implemented [`StorageList`][frame_support::storage::StorageList]
+//! trait.
+//!
+//! A fuzzer for testing is provided in crate `pallet-paged-list-fuzzer`.
+//!
+//! ## Examples
+//!
+//! 1. **Appending** some data to the list can happen either by [`Pallet::append_one`]:
+#![doc = docify::embed!("frame/paged-list/src/tests.rs", append_one_works)]
+//! 2. or by [`Pallet::append_many`]. This should always be preferred to repeated calls to
+//! [`Pallet::append_one`]:
+#![doc = docify::embed!("frame/paged-list/src/tests.rs", append_many_works)]
+//! 3. If you want to append many values (ie. in a loop), then best use the [`Pallet::appender`]:
+#![doc = docify::embed!("frame/paged-list/src/tests.rs", appender_works)]
+//! 4. **Iterating** over the list can be done with [`Pallet::iter`]. It uses the standard
+//! `Iterator` trait:
+#![doc = docify::embed!("frame/paged-list/src/tests.rs", iter_works)]
+//! 5. **Draining** elements happens through the [`Pallet::drain`] iterator. Note that even
+//! *peeking* a value will already remove it.
+#![doc = docify::embed!("frame/paged-list/src/tests.rs", drain_works)]
+//!
+//! ## Pallet API
+//!
+//! None. Only things to consider is the [`Config`] traits.
+//!
+//! ## Low Level / Implementation Details
+//!
+//! Implementation details are documented in [`paged_list::StoragePagedList`].  
+//! All storage entries are prefixed with a unique prefix that is generated by [`ListPrefix`].
+
+#![cfg_attr(not(feature = "std"), no_std)]
+
+pub use pallet::*;
+
+pub mod mock;
+mod paged_list;
+mod tests;
+
+use codec::FullCodec;
+use frame_support::{
+	pallet_prelude::StorageList,
+	traits::{PalletInfoAccess, StorageInstance},
+};
+pub use paged_list::StoragePagedList;
+
+#[frame_support::pallet]
+pub mod pallet {
+	use super::*;
+	use frame_support::pallet_prelude::*;
+
+	#[pallet::pallet]
+	pub struct Pallet<T, I = ()>(_);
+
+	#[pallet::config]
+	pub trait Config<I: 'static = ()>: frame_system::Config {
+		/// The value type that can be stored in the list.
+		type Value: FullCodec;
+
+		/// The number of values that can be put into newly created pages.
+		///
+		/// Note that this does not retroactively affect already created pages. This value can be
+		/// changed at any time without requiring a runtime migration.
+		#[pallet::constant]
+		type ValuesPerNewPage: Get<u32>;
+	}
+
+	/// A storage paged list akin to what the FRAME macros would generate.
+	// Note that FRAME does natively support paged lists in storage.
+	pub type List<T, I> = StoragePagedList<
+		ListPrefix<T, I>,
+		<T as Config<I>>::Value,
+		<T as Config<I>>::ValuesPerNewPage,
+	>;
+}
+
+// This exposes the list functionality to other pallets.
+impl<T: Config<I>, I: 'static> StorageList<T::Value> for Pallet<T, I> {
+	type Iterator = <List<T, I> as StorageList<T::Value>>::Iterator;
+	type Appender = <List<T, I> as StorageList<T::Value>>::Appender;
+
+	fn iter() -> Self::Iterator {
+		List::<T, I>::iter()
+	}
+
+	fn drain() -> Self::Iterator {
+		List::<T, I>::drain()
+	}
+
+	fn appender() -> Self::Appender {
+		List::<T, I>::appender()
+	}
+}
+
+/// Generates a unique storage prefix for each instance of the pallet.
+pub struct ListPrefix<T, I>(core::marker::PhantomData<(T, I)>);
+
+impl<T: Config<I>, I: 'static> StorageInstance for ListPrefix<T, I> {
+	fn pallet_prefix() -> &'static str {
+		crate::Pallet::<T, I>::name()
+	}
+
+	const STORAGE_PREFIX: &'static str = "paged_list";
+}