Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.

Commit e6e8da7

Browse files
authored
cli: introduce host-perf-check command (#4342)
1 parent 4663534 commit e6e8da7

File tree

16 files changed

+417
-17
lines changed

16 files changed

+417
-17
lines changed

Cargo.lock

+16
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ members = [
8484
"node/metrics",
8585
"node/metered-channel",
8686
"node/test/client",
87+
"node/test/performance-test",
8788
"node/test/service",
8889
"node/test/polkadot-simnet/common",
8990
"node/test/polkadot-simnet/node",

cli/Cargo.toml

+4
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,14 @@ futures = "0.3.17"
2121

2222
service = { package = "polkadot-service", path = "../node/service", default-features = false, optional = true }
2323
polkadot-node-core-pvf = { path = "../node/core/pvf", optional = true }
24+
polkadot-performance-test = { path = "../node/test/performance-test", optional = true }
2425

2526
sp-core = { git = "https://github.com/paritytech/substrate", branch = "master" }
2627
frame-benchmarking-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
2728
try-runtime-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
2829
sc-cli = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
2930
sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
31+
sc-tracing = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
3032

3133
# this crate is used only to enable `trie-memory-tracker` feature
3234
# see https://github.com/paritytech/substrate/pull/6745
@@ -43,9 +45,11 @@ cli = [
4345
"structopt",
4446
"sc-cli",
4547
"sc-service",
48+
"sc-tracing",
4649
"frame-benchmarking-cli",
4750
"try-runtime-cli",
4851
"polkadot-node-core-pvf",
52+
"polkadot-performance-test",
4953
]
5054
runtime-benchmarks = [ "service/runtime-benchmarks" ]
5155
trie-memory-tracker = [ "sp-trie/memory-tracker" ]

cli/build.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2015-2020 Parity Technologies (UK) Ltd.
1+
// Copyright 2017-2021 Parity Technologies (UK) Ltd.
22
// This file is part of Polkadot.
33

44
// Polkadot is free software: you can redistribute it and/or modify
@@ -15,5 +15,8 @@
1515
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
1616

1717
fn main() {
18+
if let Ok(profile) = std::env::var("PROFILE") {
19+
println!("cargo:rustc-cfg=build_type=\"{}\"", profile);
20+
}
1821
substrate_build_script_utils::generate_cargo_keys();
1922
}

cli/src/cli.rs

+4
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,10 @@ pub enum Subcommand {
5454
#[structopt(name = "benchmark", about = "Benchmark runtime pallets.")]
5555
Benchmark(frame_benchmarking_cli::BenchmarkCmd),
5656

57+
/// Runs performance checks such as PVF compilation in order to measure machine
58+
/// capabilities of running a validator.
59+
HostPerfCheck,
60+
5761
/// Try some command against runtime state.
5862
#[cfg(feature = "try-runtime")]
5963
TryRuntime(try_runtime_cli::TryRuntimeCmd),

cli/src/command.rs

+23-14
Original file line numberDiff line numberDiff line change
@@ -21,20 +21,8 @@ use sc_cli::{Role, RuntimeVersion, SubstrateCli};
2121
use service::{self, IdentifyVariant};
2222
use sp_core::crypto::Ss58AddressFormatRegistry;
2323

24-
#[derive(thiserror::Error, Debug)]
25-
pub enum Error {
26-
#[error(transparent)]
27-
PolkadotService(#[from] service::Error),
28-
29-
#[error(transparent)]
30-
SubstrateCli(#[from] sc_cli::Error),
31-
32-
#[error(transparent)]
33-
SubstrateService(#[from] sc_service::Error),
34-
35-
#[error("Other: {0}")]
36-
Other(String),
37-
}
24+
pub use crate::error::Error;
25+
pub use polkadot_performance_test::PerfCheckError;
3826

3927
impl std::convert::From<String> for Error {
4028
fn from(s: String) -> Self {
@@ -215,6 +203,20 @@ fn ensure_dev(spec: &Box<dyn service::ChainSpec>) -> std::result::Result<(), Str
215203
}
216204
}
217205

206+
/// Runs performance checks.
207+
/// Should only be used in release build since the check would take too much time otherwise.
208+
fn host_perf_check() -> Result<()> {
209+
#[cfg(not(build_type = "release"))]
210+
{
211+
Err(PerfCheckError::WrongBuildType.into())
212+
}
213+
#[cfg(build_type = "release")]
214+
{
215+
crate::host_perf_check::host_perf_check()?;
216+
Ok(())
217+
}
218+
}
219+
218220
/// Launch a node, accepting arguments just like a regular node,
219221
/// accepts an alternative overseer generator, to adjust behavior
220222
/// for integration tests as needed.
@@ -415,6 +417,13 @@ pub fn run() -> Result<()> {
415417
#[cfg(not(feature = "polkadot-native"))]
416418
panic!("No runtime feature (polkadot, kusama, westend, rococo) is enabled")
417419
},
420+
Some(Subcommand::HostPerfCheck) => {
421+
let mut builder = sc_cli::LoggerBuilder::new("");
422+
builder.with_colors(true);
423+
builder.init()?;
424+
425+
host_perf_check()
426+
},
418427
Some(Subcommand::Key(cmd)) => Ok(cmd.run(&cli)?),
419428
#[cfg(feature = "try-runtime")]
420429
Some(Subcommand::TryRuntime(cmd)) => {

cli/src/error.rs

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
// Copyright 2017-2021 Parity Technologies (UK) Ltd.
2+
// This file is part of Polkadot.
3+
4+
// Polkadot is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// Polkadot is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
14+
// You should have received a copy of the GNU General Public License
15+
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
16+
17+
#[derive(thiserror::Error, Debug)]
18+
pub enum Error {
19+
#[error(transparent)]
20+
PolkadotService(#[from] service::Error),
21+
22+
#[error(transparent)]
23+
SubstrateCli(#[from] sc_cli::Error),
24+
25+
#[error(transparent)]
26+
SubstrateService(#[from] sc_service::Error),
27+
28+
#[error(transparent)]
29+
SubstrateTracing(#[from] sc_tracing::logging::Error),
30+
31+
#[error(transparent)]
32+
PerfCheck(#[from] polkadot_performance_test::PerfCheckError),
33+
34+
#[error("Other: {0}")]
35+
Other(String),
36+
}

cli/src/host_perf_check.rs

+67
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
// Copyright 2017-2021 Parity Technologies (UK) Ltd.
2+
// This file is part of Polkadot.
3+
4+
// Polkadot is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// Polkadot is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
14+
// You should have received a copy of the GNU General Public License
15+
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
16+
17+
use log::info;
18+
use polkadot_node_core_pvf::sp_maybe_compressed_blob;
19+
use polkadot_performance_test::{
20+
measure_erasure_coding, measure_pvf_prepare, PerfCheckError, ERASURE_CODING_N_VALIDATORS,
21+
ERASURE_CODING_TIME_LIMIT, PVF_PREPARE_TIME_LIMIT, VALIDATION_CODE_BOMB_LIMIT,
22+
};
23+
use std::time::Duration;
24+
25+
pub fn host_perf_check() -> Result<(), PerfCheckError> {
26+
let wasm_code =
27+
polkadot_performance_test::WASM_BINARY.ok_or(PerfCheckError::WasmBinaryMissing)?;
28+
29+
// Decompress the code before running checks.
30+
let code = sp_maybe_compressed_blob::decompress(wasm_code, VALIDATION_CODE_BOMB_LIMIT)
31+
.or(Err(PerfCheckError::CodeDecompressionFailed))?;
32+
33+
info!("Running the performance checks...");
34+
35+
perf_check("PVF-prepare", PVF_PREPARE_TIME_LIMIT, || measure_pvf_prepare(code.as_ref()))?;
36+
37+
perf_check("Erasure-coding", ERASURE_CODING_TIME_LIMIT, || {
38+
measure_erasure_coding(ERASURE_CODING_N_VALIDATORS, code.as_ref())
39+
})?;
40+
41+
Ok(())
42+
}
43+
44+
fn green_threshold(duration: Duration) -> Duration {
45+
duration * 4 / 5
46+
}
47+
48+
fn perf_check(
49+
test_name: &str,
50+
time_limit: Duration,
51+
test: impl Fn() -> Result<Duration, PerfCheckError>,
52+
) -> Result<(), PerfCheckError> {
53+
let elapsed = test()?;
54+
55+
if elapsed < green_threshold(time_limit) {
56+
info!("🟢 {} performance check passed, elapsed: {:?}", test_name, elapsed);
57+
Ok(())
58+
} else if elapsed <= time_limit {
59+
info!(
60+
"🟡 {} performance check passed, {:?} limit almost exceeded, elapsed: {:?}",
61+
test_name, time_limit, elapsed
62+
);
63+
Ok(())
64+
} else {
65+
Err(PerfCheckError::TimeOut { elapsed, limit: time_limit })
66+
}
67+
}

cli/src/lib.rs

+4
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
mod cli;
2323
#[cfg(feature = "cli")]
2424
mod command;
25+
#[cfg(feature = "cli")]
26+
mod error;
27+
#[cfg(all(feature = "cli", build_type = "release"))]
28+
mod host_perf_check;
2529

2630
#[cfg(feature = "full-node")]
2731
pub use service::RuntimeApiCollection;

erasure-coding/src/lib.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ pub fn obtain_chunks_v1(n_validators: usize, data: &AvailableData) -> Result<Vec
135135
/// Obtain erasure-coded chunks, one for each validator.
136136
///
137137
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
138-
fn obtain_chunks<T: Encode>(n_validators: usize, data: &T) -> Result<Vec<Vec<u8>>, Error> {
138+
pub fn obtain_chunks<T: Encode>(n_validators: usize, data: &T) -> Result<Vec<Vec<u8>>, Error> {
139139
let params = code_params(n_validators)?;
140140
let encoded = data.encode();
141141

@@ -186,7 +186,7 @@ where
186186
/// are provided, recovery is not possible.
187187
///
188188
/// Works only up to 65536 validators, and `n_validators` must be non-zero.
189-
fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result<T, Error>
189+
pub fn reconstruct<'a, I: 'a, T: Decode>(n_validators: usize, chunks: I) -> Result<T, Error>
190190
where
191191
I: IntoIterator<Item = (&'a [u8], usize)>,
192192
{

node/core/pvf/src/lib.rs

+5
Original file line numberDiff line numberDiff line change
@@ -102,4 +102,9 @@ pub use metrics::Metrics;
102102
pub use execute::worker_entrypoint as execute_worker_entrypoint;
103103
pub use prepare::worker_entrypoint as prepare_worker_entrypoint;
104104

105+
pub use executor_intf::{prepare, prevalidate};
106+
107+
pub use sc_executor_common;
108+
pub use sp_maybe_compressed_blob;
109+
105110
const LOG_TARGET: &str = "parachain::pvf";

node/test/performance-test/Cargo.toml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
[package]
2+
name = "polkadot-performance-test"
3+
version = "0.9.13"
4+
authors = ["Parity Technologies <admin@parity.io>"]
5+
edition = "2018"
6+
7+
[dependencies]
8+
thiserror = "1.0.30"
9+
quote = "1.0.10"
10+
env_logger = "0.9"
11+
log = "0.4"
12+
13+
polkadot-node-core-pvf = { path = "../../core/pvf" }
14+
polkadot-erasure-coding = { path = "../../../erasure-coding" }
15+
polkadot-node-primitives = { path = "../../primitives" }
16+
17+
kusama-runtime = { path = "../../../runtime/kusama" }
18+
19+
[[bin]]
20+
name = "gen-ref-constants"
21+
path = "src/gen_ref_constants.rs"

node/test/performance-test/build.rs

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
// Copyright 2017-2021 Parity Technologies (UK) Ltd.
2+
// This file is part of Polkadot.
3+
4+
// Polkadot is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// Polkadot is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
14+
// You should have received a copy of the GNU General Public License
15+
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
16+
17+
fn main() {
18+
if let Ok(profile) = std::env::var("PROFILE") {
19+
println!("cargo:rustc-cfg=build_type=\"{}\"", profile);
20+
}
21+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
// Copyright 2017-2021 Parity Technologies (UK) Ltd.
2+
// This file is part of Polkadot.
3+
4+
// Polkadot is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
9+
// Polkadot is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU General Public License for more details.
13+
14+
// You should have received a copy of the GNU General Public License
15+
// along with Polkadot. If not, see <http://www.gnu.org/licenses/>.
16+
17+
//! This file was automatically generated by `gen-ref-constants`.
18+
//! Do not edit manually!
19+
20+
use std::time::Duration;
21+
pub const PVF_PREPARE_TIME_LIMIT: Duration = Duration::from_millis(4910u64);
22+
pub const ERASURE_CODING_TIME_LIMIT: Duration = Duration::from_millis(466u64);

0 commit comments

Comments
 (0)