Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for new extension benchmarking #75

Open
wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Add support for new extension benchmarking #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
aa987e1
Update bench-all-substrate
georgepisaltu Mar 1, 2024
066a0d2
Update bench-all-substrate
georgepisaltu Mar 1, 2024
de1e5a4
Fix `tasks-example` pallet bench
georgepisaltu Mar 1, 2024
3e7b882
Support asset conversion ops
georgepisaltu Oct 2, 2024
5d15cc0
Remove riscv feature from bench-all polkadot
georgepisaltu Oct 2, 2024
61504c4
Update bench-pallet.sh
mordamax Oct 22, 2024
cf43c11
Update bench.cmd.json
mordamax Oct 22, 2024
3364023
Update bench-pallet.sh
mordamax Oct 22, 2024
709b59c
Merge branch 'main' into george/substrate-ext
georgepisaltu Nov 7, 2024
3769ba8
Merge remote-tracking branch 'origin/mak-machinetest' into george/sub…
georgepisaltu Nov 7, 2024
58d6cf6
Remove `riscv` feature from substrate bench-all
georgepisaltu Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions commands/bench-all/bench-all.cmd.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
"repos": ["polkadot-sdk"],
"args": {
"target_dir": { "label": "Target Directory", "type_string": "substrate" },
"features": { "label": "Additional Features", "type_string": "riscv" }
"features": { "label": "Additional Features", "type_string": "" }
}
},
"polkadot": {
Expand All @@ -35,7 +35,7 @@
"args": {
"runtime": { "label": "Runtime", "type_one_of": ["rococo", "westend"] },
"target_dir": { "label": "Target Directory", "type_string":"polkadot" },
"features": { "label": "Additional Features", "type_string": "riscv" }
"features": { "label": "Additional Features", "type_string": "" }
}
},
"cumulus": {
Expand Down
33 changes: 22 additions & 11 deletions commands/bench/bench.cmd.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
"runtime": { "label": "Runtime", "type_one_of": ["dev"] },
"pallet": { "label": "pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"target_dir": { "label": "Target Directory", "type_string": "substrate" },
"features": { "label": "AdditionalFeatures", "type_string": "riscv" }
"features": { "label": "AdditionalFeatures", "type_string": "riscv" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"polkadot-pallet": {
Expand All @@ -30,7 +31,8 @@
"runtime": { "label": "Runtime", "type_one_of": ["rococo", "westend"] },
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"target_dir": { "label": "Target Directory", "type_string": "polkadot" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-assets": {
Expand All @@ -42,7 +44,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "assets" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-collectives": {
Expand All @@ -54,7 +57,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "collectives" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-coretime": {
Expand All @@ -66,7 +70,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "coretime" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-bridge-hubs": {
Expand All @@ -78,7 +83,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "bridge-hubs" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-contracts": {
Expand All @@ -90,7 +96,8 @@
"runtime_dir": { "label": "Runtime Dir", "type_string": "contracts" },
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-glutton": {
Expand All @@ -102,7 +109,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "glutton" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-starters": {
Expand All @@ -114,7 +122,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "starters" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-people": {
Expand All @@ -126,7 +135,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "people" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
},
"cumulus-testing": {
Expand All @@ -138,7 +148,8 @@
"pallet": { "label": "Pallet", "type_rule": "^([a-z_]+)([:]{2}[a-z_]+)?$", "example": "pallet_name" },
"runtime_dir": { "label": "Runtime Dir", "type_string": "testing" },
"target_dir": { "label": "Target Directory", "type_string": "cumulus" },
"features": { "label": "Additional Features", "type_string": "" }
"features": { "label": "Additional Features", "type_string": "" },
"machine": { "label": "Machine", "type_string": "" }
}
}
}
Expand Down
19 changes: 19 additions & 0 deletions commands/bench/lib/bench-all-substrate.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ EXCLUDED_PALLETS=(
"pallet_offences"
# Only used for testing, does not need real weights.
"frame_benchmarking_pallet_pov"
"pallet_example_authorization_tx_extension"
"pallet_example_tasks"
"pallet_example_basic"
"pallet_example_split"
Expand Down Expand Up @@ -120,6 +121,24 @@ for PALLET in "${ALL_PALLETS[@]}"; do
continue
fi

# Special handling of custom weight paths.
if [ "$PALLET" == "frame_system_extensions" ] || [ "$PALLET" == "frame-system-extensions" ]
then
WEIGHT_FILE="$output_path/frame/system/src/extensions/weights.rs"
elif [ "$PALLET" == "pallet_asset_conversion_tx_payment" ] || [ "$PALLET" == "pallet-asset-conversion-tx-payment" ]
then
WEIGHT_FILE="$output_path/frame/transaction-payment/asset-conversion-tx-payment/src/weights.rs"
elif [ "$PALLET" == "pallet_asset_tx_payment" ] || [ "$PALLET" == "pallet-asset-tx-payment" ]
then
WEIGHT_FILE="$output_path/frame/transaction-payment/asset-tx-payment/src/weights.rs"
elif [ "$PALLET" == "tasks_example" ] || [ "$PALLET" == "tasks-example" ]
then
WEIGHT_FILE="$output_path/frame/examples/tasks/src/weights.rs"
elif [ "$PALLET" == "pallet_asset_conversion_ops" ] || [ "$PALLET" == "pallet-asset-conversion-ops" ]
then
WEIGHT_FILE="$output_path/frame/asset-conversion/ops/src/weights.rs"
fi

echo "[+] Benchmarking $PALLET with weight file $WEIGHT_FILE";

set +e # Disable exit on error for the benchmarking of the pallets
Expand Down
15 changes: 15 additions & 0 deletions commands/bench/lib/bench-pallet.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ bench_pallet() {
get_arg optional --features "$@"
local additional_features="${out:-""}"

get_arg optional --machine "$@"
local machine="${out:-""}"

local features="runtime-benchmarks"
if [ -n "$additional_features" ]; then
features+=",$additional_features"
Expand Down Expand Up @@ -73,6 +76,10 @@ bench_pallet() {
die "Subcommand $subcommand is not supported for $target_dir in bench_pallet"
;;
esac

if [ -n "$machine" ]; then
$cargo_run_benchmarks --features="$features" -- benchmark machine --chain="$runtime" --allow-fail
fi
;;
polkadot)
# For backward compatibility: replace "-dev" with ""
Expand Down Expand Up @@ -106,6 +113,10 @@ bench_pallet() {
die "Subcommand $subcommand is not supported for $target_dir in bench_pallet"
;;
esac

if [ -n "$machine" ]; then
$cargo_run_benchmarks --bin=polkadot --features="$features" -- benchmark machine --chain="$runtime-dev" --allow-fail
fi
;;
cumulus)
get_arg required --runtime_dir "$@"
Expand Down Expand Up @@ -147,6 +158,10 @@ bench_pallet() {
die "Subcommand $subcommand is not supported for $target_dir in bench_pallet"
;;
esac

if [ -n "$machine" ]; then
$cargo_run_benchmarks -p=polkadot-parachain-bin --features="$features" -- benchmark machine --chain="$chain" --allow-fail
fi
;;
*)
die "Repository $target_dir is not supported in bench_pallet"
Expand Down