Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Co #11164: Sub-commands for benchmark #5247

Merged
merged 16 commits into from
Apr 7, 2022
Merged

Co #11164: Sub-commands for benchmark #5247

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
9eb70ab
Restructure benchmark commands
ggwpez Apr 4, 2022
b705d56
Add benchmark storage test
ggwpez Apr 4, 2022
4a2a77e
Merge remote-tracking branch 'origin/master' into oty-restructure-bench
ggwpez Apr 6, 2022
c321b57
cargo update -p sp-io
shawntabrizi Apr 6, 2022
a4f149b
Revert "cargo update -p sp-io"
ggwpez Apr 7, 2022
dc72f21
Merge remote-tracking branch 'origin/master' into oty-restructure-bench
ggwpez Apr 7, 2022
f94d7e8
cargo update -p sp-io
ggwpez Apr 7, 2022
0d32d9c
update lockfile for {"substrate"}
Apr 7, 2022
5143ee1
remove `useless_attribute`
shawntabrizi Apr 7, 2022
0c4a205
Remove rococo-native cfg
ggwpez Apr 7, 2022
45940ea
Make cumulus build
ggwpez Apr 7, 2022
a01dc00
Fix benchmark commands for the CI
ggwpez Apr 7, 2022
72617f3
Make cumulus build
ggwpez Apr 7, 2022
0bd2ce0
Make cumulus build
ggwpez Apr 7, 2022
595342a
Disable new bench commands for rococo
ggwpez Apr 7, 2022
6c683e7
Make cumulus build
ggwpez Apr 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 3 additions & 20 deletions cli/src/cli.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,28 +50,11 @@ pub enum Subcommand {
#[clap(name = "execute-worker", hide = true)]
PvfExecuteWorker(ValidationWorkerCommand),

/// The custom benchmark subcommand benchmarking runtime pallets.
#[clap(name = "benchmark", about = "Benchmark runtime pallets.")]
/// Sub-commands concerned with benchmarking.
/// The pallet benchmarking moved to the `pallet` sub-command.
#[clap(subcommand)]
Benchmark(frame_benchmarking_cli::BenchmarkCmd),

/// Benchmark the execution time of historic blocks and compare it to their consumed weight.
#[clap(
name = "benchmark-block",
about = "Benchmark the execution time of historic blocks and compare it to their consumed weight."
)]
BenchmarkBlock(frame_benchmarking_cli::BlockCmd),

/// Sub command for benchmarking the per-block and per-extrinsic execution overhead.
#[clap(
name = "benchmark-overhead",
about = "Benchmark the per-block and per-extrinsic execution overhead."
)]
BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd),

/// Sub command for benchmarking the storage speed.
#[clap(name = "benchmark-storage", about = "Benchmark storage speed.")]
BenchmarkStorage(frame_benchmarking_cli::StorageCmd),

/// Runs performance checks such as PVF compilation in order to measure machine
/// capabilities of running a validator.
HostPerfCheck,
Expand Down
281 changes: 83 additions & 198 deletions cli/src/command.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.

use crate::cli::{Cli, Subcommand};
use frame_benchmarking_cli::BenchmarkCmd;
use futures::future::TryFutureExt;
use log::info;
use sc_cli::{Role, RuntimeVersion, SubstrateCli};
Expand Down Expand Up @@ -216,6 +217,25 @@ fn ensure_dev(spec: &Box<dyn service::ChainSpec>) -> std::result::Result<(), Str
}
}

/// Unwraps a [`polkadot_client::Client`] into the concrete runtime client.
macro_rules! unwrap_client {
(
$client:ident,
$code:expr
) => {
match &*$client {
#[cfg(feature = "polkadot-native")]
polkadot_client::Client::Polkadot($client) => $code,
#[cfg(feature = "westend-native")]
polkadot_client::Client::Westend($client) => $code,
#[cfg(feature = "kusama-native")]
polkadot_client::Client::Kusama($client) => $code,
#[cfg(feature = "rococo-native")]
polkadot_client::Client::Rococo($client) => $code,
}
};
}

/// Runs performance checks.
/// Should only be used in release build since the check would take too much time otherwise.
fn host_perf_check() -> Result<()> {
Expand Down Expand Up @@ -438,212 +458,77 @@ pub fn run() -> Result<()> {
Some(Subcommand::Benchmark(cmd)) => {
let runner = cli.create_runner(cmd)?;
let chain_spec = &runner.config().chain_spec;
set_default_ss58_version(chain_spec);

ensure_dev(chain_spec).map_err(Error::Other)?;

#[cfg(feature = "kusama-native")]
if chain_spec.is_kusama() {
return Ok(runner.sync_run(|config| {
cmd.run::<service::kusama_runtime::Block, service::KusamaExecutorDispatch>(
config,
)
.map_err(|e| Error::SubstrateCli(e))
})?)
}

#[cfg(feature = "westend-native")]
if chain_spec.is_westend() {
return Ok(runner.sync_run(|config| {
cmd.run::<service::westend_runtime::Block, service::WestendExecutorDispatch>(
config,
)
.map_err(|e| Error::SubstrateCli(e))
})?)
}

// else we assume it is polkadot.
#[cfg(feature = "polkadot-native")]
{
return Ok(runner.sync_run(|config| {
cmd.run::<service::polkadot_runtime::Block, service::PolkadotExecutorDispatch>(
config,
)
.map_err(|e| Error::SubstrateCli(e))
})?)
}
#[cfg(not(feature = "polkadot-native"))]
panic!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
},
Some(Subcommand::BenchmarkBlock(cmd)) => {
let runner = cli.create_runner(cmd)?;
let chain_spec = &runner.config().chain_spec;

#[cfg(feature = "rococo-native")]
if chain_spec.is_rococo() || chain_spec.is_wococo() || chain_spec.is_versi() {
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

if let polkadot_client::Client::Rococo(pd) = &*client {
Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
} else {
unreachable!("Checked above; qed")
}
})?)
}

#[cfg(feature = "kusama-native")]
if chain_spec.is_kusama() {
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

if let polkadot_client::Client::Kusama(pd) = &*client {
Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
} else {
unreachable!("Checked above; qed")
}
})?)
}

#[cfg(feature = "westend-native")]
if chain_spec.is_westend() {
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

if let polkadot_client::Client::Westend(pd) = &*client {
Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
} else {
unreachable!("Checked above; qed")
}
})?)
}

#[cfg(feature = "polkadot-native")]
{
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

if let polkadot_client::Client::Polkadot(pd) = &*client {
Ok((cmd.run(pd.clone()).map_err(Error::SubstrateCli), task_manager))
} else {
unreachable!("Checked above; qed")
}
})?)
}

#[cfg(not(feature = "polkadot-native"))]
unreachable!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
},
Some(Subcommand::BenchmarkOverhead(cmd)) => {
use polkadot_client::benchmark_inherent_data;

let runner = cli.create_runner(cmd)?;
let chain_spec = &runner.config().chain_spec;
set_default_ss58_version(chain_spec);
ensure_dev(chain_spec).map_err(Error::Other)?;

#[cfg(feature = "rococo-native")]
if chain_spec.is_rococo() || chain_spec.is_wococo() || chain_spec.is_versi() {
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
let inherent_data = benchmark_inherent_data(header)
.map_err(|e| format!("generating inherent data: {:?}", e))?;

if let polkadot_client::Client::Rococo(pd) = &*client {
Ok((
cmd.run(config, pd.clone(), inherent_data, client)
.map_err(Error::SubstrateCli),
task_manager,
))
} else {
unreachable!("Checked above; qed")
match cmd {
BenchmarkCmd::Storage(cmd) => runner.sync_run(|mut config| {
let (client, backend, _, _) = service::new_chain_ops(&mut config, None)?;
let db = backend.expose_db();
let storage = backend.expose_storage();

unwrap_client!(client, cmd.run(config, client.clone(), db, storage))
.map_err(Error::SubstrateCli)
}),
BenchmarkCmd::Block(cmd) => runner.sync_run(|mut config| {
let (client, _, _, _) = service::new_chain_ops(&mut config, None)?;

unwrap_client!(client, cmd.run(client.clone())).map_err(Error::SubstrateCli)
}),
BenchmarkCmd::Overhead(cmd) => {
ensure_dev(chain_spec).map_err(Error::Other)?;
runner.sync_run(|mut config| {
use polkadot_client::benchmark_inherent_data;
let (client, _, _, _) = service::new_chain_ops(&mut config, None)?;
let wrapped = client.clone();

let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
let inherent_data = benchmark_inherent_data(header)
.map_err(|e| format!("generating inherent data: {:?}", e))?;

unwrap_client!(
client,
cmd.run(config, client.clone(), inherent_data, wrapped)
)
.map_err(Error::SubstrateCli)
})
},
BenchmarkCmd::Pallet(cmd) => {
set_default_ss58_version(chain_spec);
ensure_dev(chain_spec).map_err(Error::Other)?;

#[cfg(feature = "kusama-native")]
if chain_spec.is_kusama() {
return Ok(runner.sync_run(|config| {
cmd.run::<service::kusama_runtime::Block, service::KusamaExecutorDispatch>(config)
.map_err(|e| Error::SubstrateCli(e))
})?)
}
})?)
}

#[cfg(feature = "kusama-native")]
if chain_spec.is_kusama() {
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
let inherent_data = benchmark_inherent_data(header)
.map_err(|e| format!("generating inherent data: {:?}", e))?;

if let polkadot_client::Client::Kusama(pd) = &*client {
Ok((
cmd.run(config, pd.clone(), inherent_data, client)
.map_err(Error::SubstrateCli),
task_manager,
))
} else {
unreachable!("Checked above; qed")
#[cfg(feature = "westend-native")]
if chain_spec.is_westend() {
return Ok(runner.sync_run(|config| {
cmd.run::<service::westend_runtime::Block, service::WestendExecutorDispatch>(config)
.map_err(|e| Error::SubstrateCli(e))
})?)
}
})?)
}

#[cfg(feature = "westend-native")]
if chain_spec.is_westend() {
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
let inherent_data = benchmark_inherent_data(header)
.map_err(|e| format!("generating inherent data: {:?}", e))?;

if let polkadot_client::Client::Westend(pd) = &*client {
Ok((
cmd.run(config, pd.clone(), inherent_data, client)
.map_err(Error::SubstrateCli),
task_manager,
))
} else {
unreachable!("Checked above; qed")
// else we assume it is polkadot.
#[cfg(feature = "polkadot-native")]
{
return Ok(runner.sync_run(|config| {
cmd.run::<service::polkadot_runtime::Block, service::PolkadotExecutorDispatch>(config)
.map_err(|e| Error::SubstrateCli(e))
})?)
}
})?)
}

#[cfg(feature = "polkadot-native")]
{
return Ok(runner.async_run(|mut config| {
let (client, _, _, task_manager) = service::new_chain_ops(&mut config, None)?;

let header = client.header(BlockId::Number(0_u32.into())).unwrap().unwrap();
let inherent_data = benchmark_inherent_data(header)
.map_err(|e| format!("generating inherent data: {:?}", e))?;

if let polkadot_client::Client::Polkadot(pd) = &*client {
Ok((
cmd.run(config, pd.clone(), inherent_data, client)
.map_err(Error::SubstrateCli),
task_manager,
))
} else {
unreachable!("Checked above; qed")
}
})?)
#[cfg(not(feature = "polkadot-native"))]
#[allow(unreachable_code)]
Err(service::Error::NoRuntime.into())
},
// NOTE: this allows the Polkadot client to leniently implement
// new benchmark commands.
#[allow(unreachable_patterns, useless_attribute)]
_ => Err(Error::CommandNotImplemented),
}

#[cfg(not(feature = "polkadot-native"))]
unreachable!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
},
Some(Subcommand::BenchmarkStorage(cmd)) => {
let runner = cli.create_runner(cmd)?;
let chain_spec = &runner.config().chain_spec;
set_default_ss58_version(chain_spec);

Ok(runner.async_run(|mut config| {
let (client, backend, _, task_manager) = service::new_chain_ops(&mut config, None)?;
let db = backend.expose_db();
let storage = backend.expose_storage();

Ok((
cmd.run(config, client, db, storage).map_err(Error::SubstrateCli),
task_manager,
))
})?)
},
Some(Subcommand::HostPerfCheck) => {
let mut builder = sc_cli::LoggerBuilder::new("");
Expand Down
3 changes: 3 additions & 0 deletions cli/src/error.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ pub enum Error {
#[error("URL did not resolve to anything")]
AddressResolutionMissing,

#[error("Command is not implemented")]
CommandNotImplemented,

#[error("Other: {0}")]
Other(String),
}
Loading