lib.rs

// Copyright 2020 Parity Technologies (UK) Ltd.
// This file is part of Substrate.

// Substrate is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Substrate is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Substrate.  If not, see <http://www.gnu.org/licenses/>.

//! Macro for benchmarking a FRAME runtime.

#![cfg_attr(not(feature = "std"), no_std)]

mod tests;
mod utils;
pub use utils::*;
#[doc(hidden)]
pub use sp_io::storage::root as storage_root;
pub use sp_runtime::traits::Dispatchable;

/// Construct pallet benchmarks for weighing dispatchables.
///
/// Works around the idea of complexity parameters, named by a single letter (which is usually
/// upper cased in complexity notation but is lower-cased for use in this macro).
///
/// Complexity parameters ("parameters") have a range which is a `u32` pair. Every time a benchmark
/// is prepared and run, this parameter takes a concrete value within the range. There is an
/// associated instancing block, which is a single expression that is evaluated during
/// preparation. It may use `?` (`i.e. `return Err(...)`) to bail with a string error. Here's a
/// few examples:
///
/// ```ignore
/// // These two are equivalent:
/// let x in 0 .. 10;
/// let x in 0 .. 10 => ();
/// // This one calls a setup function and might return an error (which would be terminal).
/// let y in 0 .. 10 => setup(y)?;
/// // This one uses a code block to do lots of stuff:
/// let z in 0 .. 10 => {
///   let a = z * z / 5;
///   let b = do_something(a)?;
///   combine_into(z, b);
/// }
/// ```
///
/// Note that due to parsing restrictions, if the `from` expression is not a single token (i.e. a
/// literal or constant), then it must be parenthesised.
///
/// The macro allows for a number of "arms", each representing an individual benchmark. Using the
/// simple syntax, the associated dispatchable function maps 1:1 with the benchmark and the name of
/// the benchmark is the same as that of the associated function. However, extended syntax allows
/// for arbitrary expresions to be evaluated in a benchmark (including for example,
/// `on_initialize`).
///
/// The macro allows for common parameters whose ranges and instancing expressions may be drawn upon
/// (or not) by each arm. Syntax is available to allow for only the range to be drawn upon if
/// desired, allowing an alternative instancing expression to be given.
///
/// Each arm may also have a block of code which is run prior to any instancing and a block of code
/// which is run afterwards. All code blocks may draw upon the specific value of each parameter
/// at any time. Local variables are shared between the two pre- and post- code blocks, but do not
/// leak from the interior of any instancing expressions.
///
/// Any common parameters that are unused in an arm do not have their instancing expressions
/// evaluated.
///
/// Example:
/// ```ignore
/// benchmarks! {
///   // common parameter; just one for this example.
///   _ {
///     let l in 1 .. MAX_LENGTH => initialize_l(l);
///   }
///
///   // first dispatchable: foo; this is a user dispatchable and operates on a `u8` vector of
///   // size `l`, which we allow to be initialized as usual.
///   foo {
///     let caller = account::<T>(b"caller", 0, benchmarks_seed);
///     let l = ...;
///   }: _(Origin::Signed(caller), vec![0u8; l])
///
///   // second dispatchable: bar; this is a root dispatchable and accepts a `u8` vector of size
///   // `l`. We don't want it pre-initialized like before so we override using the `=> ()` notation.
///   // In this case, we explicitly name the call using `bar` instead of `_`.
///   bar {
///     let l = _ .. _ => ();
///   }: bar(Origin::Root, vec![0u8; l])
///
///   // third dispatchable: baz; this is a user dispatchable. It isn't dependent on length like the
///   // other two but has its own complexity `c` that needs setting up. It uses `caller` (in the
///   // pre-instancing block) within the code block. This is only allowed in the param instancers
///   // of arms. Instancers of common params cannot optimistically draw upon hypothetical variables
///   // that the arm's pre-instancing code block might have declared.
///   baz1 {
///     let caller = account::<T>(b"caller", 0, benchmarks_seed);
///     let c = 0 .. 10 => setup_c(&caller, c);
///   }: baz(Origin::Signed(caller))
///
///   // this is a second benchmark of the baz dispatchable with a different setup.
///   baz2 {
///     let caller = account::<T>(b"caller", 0, benchmarks_seed);
///     let c = 0 .. 10 => setup_c_in_some_other_way(&caller, c);
///   }: baz(Origin::Signed(caller))
///
///   // this is benchmarking some code that is not a dispatchable.
///   populate_a_set {
///     let x in 0 .. 10_000;
///     let mut m = Vec::<u32>::new();
///     for i in 0..x {
///       m.insert(i);
///     }
///   }: { m.into_iter().collect::<BTreeSet>() }
/// }
/// ```
#[macro_export]
macro_rules! benchmarks {
	(
		_ {
			$(
				let $common:ident in $common_from:tt .. $common_to:expr => $common_instancer:expr;
			)*
		}
		$( $rest:tt )*
	) => {
		$crate::benchmarks_iter!({
			$( { $common , $common_from , $common_to , $common_instancer } )*
		} ( ) $( $rest )* );
	}
}

#[macro_export]
#[allow(missing_docs)]
macro_rules! benchmarks_iter {
	// mutation arm:
	(
		{ $( $common:tt )* }
		( $( $names:ident )* )
		$name:ident { $( $code:tt )* }: _ ( $origin:expr $( , $arg:expr )* )
		$( $rest:tt )*
	) => {
		$crate::benchmarks_iter! {
			{ $( $common )* } ( $( $names )* ) $name { $( $code )* }: $name ( $origin $( , $arg )* ) $( $rest )*
		}
	};
	// mutation arm:
	(
		{ $( $common:tt )* }
		( $( $names:ident )* )
		$name:ident { $( $code:tt )* }: $dispatch:ident ( $origin:expr $( , $arg:expr )* )
		$( $rest:tt )*
	) => {
		$crate::benchmarks_iter! {
			{ $( $common )* } ( $( $names )* ) $name { $( $code )* }: { 
				<Call<T> as $crate::Dispatchable>::dispatch(Call::<T>::$dispatch($($arg),*), $origin.into())?;
			} $( $rest )*
		}
	};
	// iteration arm:
	(
		{ $( $common:tt )* }
		( $( $names:ident )* )
		$name:ident { $( $code:tt )* }: $eval:block
		$( $rest:tt )*
	) => {
		$crate::benchmark_backend! {
			$name { $( $common )* } { } { $eval } { $( $code )* }
		}
		$crate::benchmarks_iter!( { $( $common )* } ( $( $names )* $name ) $( $rest )* );
	};
	// iteration-exit arm
	( { $( $common:tt )* } ( $( $names:ident )* ) ) => {
		$crate::selected_benchmark!( $( $names ),* );
		$crate::impl_benchmark!( $( $names ),* );
	}
}

#[macro_export]
#[allow(missing_docs)]
macro_rules! benchmark_backend {
	// parsing arms
	($name:ident {
		$( $common:tt )*
	} {
		$( PRE { $( $pre_parsed:tt )* } )*
	} { $eval:block } {
			let $pre_id:tt : $pre_ty:ty = $pre_ex:expr;
			$( $rest:tt )*
	} ) => {
		$crate::benchmark_backend! {
			$name { $( $common )* } {
				$( PRE { $( $pre_parsed )* } )*
				PRE { $pre_id , $pre_ty , $pre_ex }
			} { $eval } { $( $rest )* }
		}
	};
	($name:ident {
		$( $common:tt )*
	} {
		$( $parsed:tt )*
	} { $eval:block } {
		let $param:ident in ( $param_from:expr ) .. $param_to:expr => $param_instancer:expr;
		$( $rest:tt )*
	}) => {
		$crate::benchmark_backend! {
			$name { $( $common )* } {
				$( $parsed )*
				PARAM { $param , $param_from , $param_to , $param_instancer }
			} { $eval } { $( $rest )* }
		}
	};
	// mutation arm to look after defaulting to a common param
	($name:ident {
		$( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )*
	} {
		$( $parsed:tt )*
	} { $eval:block } {
		let $param:ident in ...;
		$( $rest:tt )*
	}) => {
		$crate::benchmark_backend! {
			$name {
				$( { $common , $common_from , $common_to , $common_instancer } )*
			} {
				$( $parsed )*
			} { $eval } {
				let $param
					in ({ $( let $common = $common_from; )* $param })
					.. ({ $( let $common = $common_to; )* $param })
					=> ({ $( let $common = || -> Result<(), &'static str> { $common_instancer ; Ok(()) }; )* $param()? });
				$( $rest )*
			}
		}
	};
	// mutation arm to look after defaulting only the range to common param
	($name:ident {
		$( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )*
	} {
		$( $parsed:tt )*
	} { $eval:block } {
		let $param:ident in _ .. _ => $param_instancer:expr ;
		$( $rest:tt )*
	}) => {
		$crate::benchmark_backend! {
			$name {
				$( { $common , $common_from , $common_to , $common_instancer } )*
			} {
				$( $parsed )*
			} { $eval } {
				let $param
					in ({ $( let $common = $common_from; )* $param })
					.. ({ $( let $common = $common_to; )* $param })
					=> $param_instancer ;
				$( $rest )*
			}
		}
	};
	// mutation arm to look after a single tt for param_from.
	($name:ident {
		$( $common:tt )*
	} {
		$( $parsed:tt )*
	} { $eval:block } {
		let $param:ident in $param_from:tt .. $param_to:expr => $param_instancer:expr ;
		$( $rest:tt )*
	}) => {
		$crate::benchmark_backend! {
			$name { $( $common )* } { $( $parsed )* } { $eval } {
				let $param in ( $param_from ) .. $param_to => $param_instancer;
				$( $rest )*
			}
		}
	};
	// mutation arm to look after the default tail of `=> ()`
	($name:ident {
		$( $common:tt )*
	} {
		$( $parsed:tt )*
	} { $eval:block } {
		let $param:ident in $param_from:tt .. $param_to:expr;
		$( $rest:tt )*
	}) => {
		$crate::benchmark_backend! {
			$name { $( $common )* } { $( $parsed )* } { $eval } {
				let $param in $param_from .. $param_to => ();
				$( $rest )*
			}
		}
	};
	// mutation arm to look after `let _ =`
	($name:ident {
		$( $common:tt )*
	} {
		$( $parsed:tt )*
	} { $eval:block } {
		let $pre_id:tt = $pre_ex:expr;
		$( $rest:tt )*
	}) => {
		$crate::benchmark_backend! {
			$name { $( $common )* } { $( $parsed )* } { $eval } {
				let $pre_id : _ = $pre_ex;
				$( $rest )*
			}
		}
	};
	// actioning arm
	($name:ident {
		$( { $common:ident , $common_from:tt , $common_to:expr , $common_instancer:expr } )*
	} {
		$( PRE { $pre_id:tt , $pre_ty:ty , $pre_ex:expr } )*
		$( PARAM { $param:ident , $param_from:expr , $param_to:expr , $param_instancer:expr } )*
	} { $eval:block } { $( $post:tt )* } ) => {
		#[allow(non_camel_case_types)]
		struct $name;
		#[allow(unused_variables)]
		impl<T: Trait> $crate::BenchmarkingSetup<T> for $name {
			fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> {
				vec! [
					$(
						($crate::BenchmarkParameter::$param, $param_from, $param_to)
					),*
				]
			}

			fn instance(&self, components: &[($crate::BenchmarkParameter, u32)])
				-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
			{
				$(
					let $common = $common_from;
				)*
				$(
					// Prepare instance
					let $param = components.iter().find(|&c| c.0 == $crate::BenchmarkParameter::$param).unwrap().1;
				)*
				$(
					let $pre_id : $pre_ty = $pre_ex;
				)*
				$( $param_instancer ; )*
				$( $post )*

				Ok(Box::new(move || -> Result<(), &'static str> { $eval; Ok(()) }))
			}
		}
	}
}

/// Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`.
///
/// Every variant must implement [`BenchmarkingSetup`].
///
/// ```nocompile
///
/// struct Transfer;
/// impl BenchmarkingSetup for Transfer { ... }
///
/// struct SetBalance;
/// impl BenchmarkingSetup for SetBalance { ... }
///
/// selected_benchmark!(Transfer, SetBalance);
/// ```
#[macro_export]
macro_rules! selected_benchmark {
	(
		$( $bench:ident ),*
	) => {
		// The list of available benchmarks for this pallet.
		#[allow(non_camel_case_types)]
		enum SelectedBenchmark {
			$( $bench, )*
		}

		// Allow us to select a benchmark from the list of available benchmarks.
		impl<T: Trait> $crate::BenchmarkingSetup<T> for SelectedBenchmark {
			fn components(&self) -> Vec<($crate::BenchmarkParameter, u32, u32)> {
				match self {
					$( Self::$bench => <$bench as $crate::BenchmarkingSetup<T>>::components(&$bench), )*
				}
			}

			fn instance(&self, components: &[($crate::BenchmarkParameter, u32)])
				-> Result<Box<dyn FnOnce() -> Result<(), &'static str>>, &'static str>
			{
				match self {
					$( Self::$bench => <$bench as $crate::BenchmarkingSetup<T>>::instance(&$bench, components), )*
				}
			}
		}
	};
}

#[macro_export]
macro_rules! impl_benchmark {
	(
		$( $name:ident ),*
	) => {
		impl<T: Trait> $crate::Benchmarking<$crate::BenchmarkResults> for Module<T> {
			fn run_benchmark(
				extrinsic: Vec<u8>,
				lowest_range_values: Vec<u32>,
				highest_range_values: Vec<u32>,
				steps: Vec<u32>,
				repeat: u32,
			) -> Result<Vec<$crate::BenchmarkResults>, &'static str> {
				// Map the input to the selected benchmark.
				let extrinsic = sp_std::str::from_utf8(extrinsic.as_slice())
					.map_err(|_| "`extrinsic` is not a valid utf8 string!")?;
				let selected_benchmark = match extrinsic {
					$( stringify!($name) => SelectedBenchmark::$name, )*
					_ => return Err("Could not find extrinsic."),
				};

				// Warm up the DB
				$crate::benchmarking::commit_db();
				$crate::benchmarking::wipe_db();

				let components = <SelectedBenchmark as $crate::BenchmarkingSetup<T>>::components(&selected_benchmark);
				let mut results: Vec<$crate::BenchmarkResults> = Vec::new();

				// Default number of steps for a component.
				let mut prev_steps = 10;

				// Select the component we will be benchmarking. Each component will be benchmarked.
				for (idx, (name, low, high)) in components.iter().enumerate() {
					// Get the number of steps for this component.
					let steps = steps.get(idx).cloned().unwrap_or(prev_steps);
					prev_steps = steps;

					let lowest = lowest_range_values.get(idx).cloned().unwrap_or(*low);
					let highest = highest_range_values.get(idx).cloned().unwrap_or(*high);

					let diff = highest - lowest;

					// Create up to `STEPS` steps for that component between high and low.
					let step_size = (diff / steps).max(1);
					let num_of_steps = diff / step_size + 1;

					for s in 0..num_of_steps {
						// This is the value we will be testing for component `name`
						let component_value = lowest + step_size * s;

						// Select the max value for all the other components.
						let c: Vec<($crate::BenchmarkParameter, u32)> = components.iter()
							.enumerate()
							.map(|(idx, (n, _, h))|
								if n == name {
									(*n, component_value)
								} else {
									(*n, *highest_range_values.get(idx).unwrap_or(h))
								}
							)
							.collect();

						// Run the benchmark `repeat` times.
						for _ in 0..repeat {
							// Set up the externalities environment for the setup we want to benchmark.
							let closure_to_benchmark = <SelectedBenchmark as $crate::BenchmarkingSetup<T>>::instance(&selected_benchmark, &c)?;

							// Commit the externalities to the database, flushing the DB cache.
							// This will enable worst case scenario for reading from the database.
							$crate::benchmarking::commit_db();

							// Time the extrinsic logic.
							let start_extrinsic = $crate::benchmarking::current_time();
							closure_to_benchmark()?;
							let finish_extrinsic = $crate::benchmarking::current_time();
							let elapsed_extrinsic = finish_extrinsic - start_extrinsic;

							// Time the storage root recalculation.
							let start_storage_root = $crate::benchmarking::current_time();
							$crate::storage_root();
							let finish_storage_root = $crate::benchmarking::current_time();
							let elapsed_storage_root = finish_storage_root - start_storage_root;

							results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root));

							// Wipe the DB back to the genesis state.
							$crate::benchmarking::wipe_db();
						}
					}
				}
				return Ok(results);
			}
		}
	}
}