Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft benchmark sanity weight test #1493

Closed
wants to merge 16 commits into from
Closed

Draft benchmark sanity weight test #1493

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
0ceee42
Draft benchmark sanity weight test
Daanvdplas Sep 5, 2023
81165e3
error || warning
Daanvdplas Oct 17, 2023
8fc1fca
allow enable, warning & ignore + output improvement
Daanvdplas Oct 18, 2023
7d64b60
Merge master
Daanvdplas Oct 18, 2023
d476d7f
fmt
Daanvdplas Oct 18, 2023
a698258
spelling & ci tests
Daanvdplas Oct 18, 2023
6081db0
fix failing ci tests
Daanvdplas Oct 18, 2023
540ce57
more ci tests
Daanvdplas Oct 18, 2023
df4f1c4
more...
Daanvdplas Oct 19, 2023
ac0bc11
Merge branch 'master' into daan/built_in_weight_test
Daanvdplas Oct 25, 2023
d64188f
add to gitlab pipeline
Daanvdplas Oct 25, 2023
eace65b
fix: unused import
Daanvdplas Oct 25, 2023
98485c5
fmt
Daanvdplas Oct 25, 2023
8795558
improve based on first comments (1/2)
Daanvdplas Oct 27, 2023
741329f
add Cargo.lock
Daanvdplas Oct 27, 2023
0f76c55
Master.into()
kianenigma Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions .gitlab/pipeline/short-benchmarks.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ short-benchmark-westend: &short-bench
tags:
- benchmark
script:
- ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
- ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --sanity-weight-check ignore

# run short-benchmarks for system parachain runtimes from cumulus

Expand All @@ -47,8 +47,7 @@ short-benchmark-westend: &short-bench
tags:
- benchmark
script:
- ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1

- ./artifacts/polkadot-parachain benchmark pallet --chain $RUNTIME_CHAIN --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --sanity-weight-check ignore
short-benchmark-asset-hub-polkadot:
<<: *short-bench-cumulus
variables:
Expand Down
4 changes: 1 addition & 3 deletions .gitlab/pipeline/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ test-linux-stable-runtime-benchmarks:
RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
script:
- time cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet

# can be used to run all tests
# test-linux-stable-all:
# stage: test
Expand Down Expand Up @@ -352,8 +351,7 @@ quick-benchmarks:
WASM_BUILD_NO_COLOR: 1
WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
script:
- time cargo run --locked --release -p node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1

- time cargo run --locked --release -p node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --execution wasm --wasm-execution compiled --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --sanity-weight-check ignore
test-frame-examples-compile-to-wasm:
# into one job
stage: test
Expand Down
6 changes: 5 additions & 1 deletion cumulus/parachain-template/runtime/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,6 +702,8 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -712,7 +714,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1162,11 +1162,14 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

// This is defined once again in dispatch_benchmark, because list_benchmarks!
// and add_benchmarks! are macros exported by define_benchmarks! macros and those types
Expand All @@ -1186,7 +1189,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1042,11 +1042,14 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

// This is defined once again in dispatch_benchmark, because list_benchmarks!
// and add_benchmarks! are macros exported by define_benchmarks! macros and those types
Expand All @@ -1065,7 +1068,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1241,12 +1241,15 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use pallet_xcm_bridge_hub_router::benchmarking::Pallet as XcmBridgeHubRouterBench;
use sp_core::Get;

// This is defined once again in dispatch_benchmark, because list_benchmarks!
// and add_benchmarks! are macros exported by define_benchmarks! macros and those types
Expand All @@ -1269,7 +1272,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1226,11 +1226,14 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

// This is defined once again in dispatch_benchmark, because list_benchmarks!
// and add_benchmarks! are macros exported by define_benchmarks! macros and those types
Expand All @@ -1250,7 +1253,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -631,11 +631,14 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

// This is defined once again in dispatch_benchmark, because list_benchmarks!
// and add_benchmarks! are macros exported by define_benchmarks! macros and those types
Expand All @@ -647,7 +650,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -631,11 +631,14 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

// This is defined once again in dispatch_benchmark, because list_benchmarks!
// and add_benchmarks! are macros exported by define_benchmarks! macros and those types
Expand All @@ -647,7 +650,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -811,11 +811,14 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

// This is defined once again in dispatch_benchmark, because list_benchmarks!
// and add_benchmarks! are macros exported by define_benchmarks! macros and those types
Expand All @@ -836,7 +839,9 @@ impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -902,17 +902,22 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

let mut list = Vec::<BenchmarkList>::new();
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,17 +645,22 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

let mut list = Vec::<BenchmarkList>::new();
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
8 changes: 6 additions & 2 deletions cumulus/parachains/runtimes/glutton/glutton-kusama/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -437,17 +437,21 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use sp_core::Get;

let mut list = Vec::<BenchmarkList>::new();
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();

(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
7 changes: 6 additions & 1 deletion cumulus/parachains/runtimes/testing/penpal/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,17 +790,22 @@ impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
use frame_system_benchmarking::Pallet as SystemBench;
use cumulus_pallet_session_benchmarking::Pallet as SessionBench;
use sp_core::Get;

let mut list = Vec::<BenchmarkList>::new();
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
(list, storage_info)
let max_extrinsic_weight = RuntimeBlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
6 changes: 5 additions & 1 deletion polkadot/runtime/rococo/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2041,6 +2041,8 @@ sp_api::impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -2052,7 +2054,9 @@ sp_api::impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
return (list, storage_info)
let max_extrinsic_weight = BlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hoped that we could somehow around doing a breaking change for all runtimes, but it seems unavoidable, unless we want to squeeze this into into any of the vectors.
Maybe you can create a single struct that holds the Weight and RuntimeDbWeight. Then we can extend that in the future and avoid more breaking changes.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Creating a single struct for Weight and RuntimeDbWeight is either way a better solution. Would you prefer to have it included in BenchmarkList / StorageInfo and make it backwards compatible or add an additional parameter which results in a breaking change?

Copy link
Member

@ggwpez ggwpez Feb 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it would need one breaking change, yes. But from then on we can avoid further breaking changes by using the following pattern:

struct AllBenchmarkData { // not a good name
  list: Vec<frame_benchmarking::BenchmarkList>,
  info: Vec<frame_support::traits::StorageInfo>,
  max_ext_weight: Option<..>,
  db_weights: Option<..>
}

and having Default on that thing. The runtime devs can use ..Default::default() to stay forward compatible and we can keep adding Options without directly breaking their code.

}

fn dispatch_benchmark(
Expand Down
6 changes: 5 additions & 1 deletion polkadot/runtime/westend/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2124,6 +2124,8 @@ sp_api::impl_runtime_apis! {
fn benchmark_metadata(extra: bool) -> (
Vec<frame_benchmarking::BenchmarkList>,
Vec<frame_support::traits::StorageInfo>,
Weight,
frame_support::weights::RuntimeDbWeight,
) {
use frame_benchmarking::{Benchmarking, BenchmarkList};
use frame_support::traits::StorageInfoTrait;
Expand All @@ -2141,7 +2143,9 @@ sp_api::impl_runtime_apis! {
list_benchmarks!(list, extra);

let storage_info = AllPalletsWithSystem::storage_info();
return (list, storage_info)
let max_extrinsic_weight = BlockWeights::get().per_class.get(DispatchClass::Normal).max_extrinsic.unwrap();
let db_weight: frame_support::weights::RuntimeDbWeight = <Self as frame_system::Config>::DbWeight::get();
(list, storage_info, max_extrinsic_weight, db_weight)
}

fn dispatch_benchmark(
Expand Down
Loading