[subsystem-benchmarks] Fix availability-write regression tests (#3698)

Adds availability-write regression tests. The results for the `availability-distribution` subsystem are volatile, so I had to reduce the precision of the test.
paritytech · Mar 25, 2024 · cc1e6ac · cc1e6ac
1 parent 9d12240
commit cc1e6ac
Show file tree

Hide file tree

Showing 14 changed files with 435 additions and 458 deletions.
diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
@@ -503,6 +503,7 @@ subsystem-regression-tests:
     - .run-immediately
   script:
     - cargo bench --profile=testnet -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks
+    - cargo bench --profile=testnet -p polkadot-availability-distribution --bench availability-distribution-regression-bench --features subsystem-benchmarks
   tags:
     - benchmark
   allow_failure: true
diff --git a/polkadot/node/network/availability-distribution/Cargo.toml b/polkadot/node/network/availability-distribution/Cargo.toml
@@ -39,9 +39,9 @@ polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" }
 polkadot-subsystem-bench = { path = "../../subsystem-bench" }
 
 
-[[test]]
+[[bench]]
 name = "availability-distribution-regression-bench"
-path = "tests/availability-distribution-regression-bench.rs"
+path = "benches/availability-distribution-regression-bench.rs"
 harness = false
 required-features = ["subsystem-benchmarks"]
 

diff --git a/...lability-distribution-regression-bench.rs → ...lability-distribution-regression-bench.rs b/...lability-distribution-regression-bench.rs → ...lability-distribution-regression-bench.rs
@@ -24,46 +24,55 @@
 //! - availability-store
 
 use polkadot_subsystem_bench::{
-	availability::{benchmark_availability_write, prepare_test, TestDataAvailability, TestState},
+	availability::{benchmark_availability_write, prepare_test, TestState},
 	configuration::TestConfiguration,
-	utils::{warm_up_and_benchmark, WarmUpOptions},
+	usage::BenchmarkUsage,
 };
+use std::io::Write;
+
+const BENCH_COUNT: usize = 50;
 
 fn main() -> Result<(), String> {
 	let mut messages = vec![];
 	let mut config = TestConfiguration::default();
-	// A single node effort roughly n_cores * needed_approvals / n_validators = 60 * 30 / 300
-	config.n_cores = 6;
+	// A single node effort roughly
+	config.n_cores = 10;
+	config.n_validators = 500;
 	config.num_blocks = 3;
 	config.generate_pov_sizes();
+	let state = TestState::new(&config);
 
-	let usage = warm_up_and_benchmark(
-		WarmUpOptions::new(&[
-			"availability-distribution",
-			"bitfield-distribution",
-			"availability-store",
-		]),
-		|| {
-			let mut state = TestState::new(&config);
-			let (mut env, _protocol_config) =
-				prepare_test(config.clone(), &mut state, TestDataAvailability::Write, false);
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> = (0..BENCH_COUNT)
+		.map(|n| {
+			print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n));
+			std::io::stdout().flush().unwrap();
+			let (mut env, _cfgs) = prepare_test(
+				&state,
+				polkadot_subsystem_bench::availability::TestDataAvailability::Write,
+				false,
+			);
 			env.runtime().block_on(benchmark_availability_write(
 				"data_availability_write",
 				&mut env,
-				state,
+				&state,
 			))
-		},
-	)?;
-	println!("{}", usage);
+		})
+		.collect();
+	println!("\rDone!{}", " ".repeat(BENCH_COUNT));
+	let average_usage = BenchmarkUsage::average(&usages);
+	println!("{}", average_usage);
 
-	messages.extend(usage.check_network_usage(&[
-		("Received from peers", 443.333, 0.05),
-		("Sent to peers", 21818.555, 0.05),
+	// We expect no variance for received and sent
+	// but use 0.001 because we operate with floats
+	messages.extend(average_usage.check_network_usage(&[
+		("Received from peers", 433.3, 0.001),
+		("Sent to peers", 18480.0, 0.001),
 	]));
-	messages.extend(usage.check_cpu_usage(&[
-		("availability-distribution", 0.011, 0.05),
-		("bitfield-distribution", 0.029, 0.05),
-		("availability-store", 0.232, 0.05),
+	messages.extend(average_usage.check_cpu_usage(&[
+		("availability-distribution", 0.012, 0.05),
+		("availability-store", 0.153, 0.05),
+		("bitfield-distribution", 0.026, 0.05),
 	]));
 
 	if messages.is_empty() {

diff --git a/...adot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs b/...adot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs
@@ -14,9 +14,9 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-//! availability-write regression tests
+//! availability-read regression tests
 //!
-//! Availability write benchmark based on Kusama parameters and scale.
+//! Availability read benchmark based on Kusama parameters and scale.
 //!
 //! Subsystems involved:
 //! - availability-recovery
@@ -27,8 +27,11 @@ use polkadot_subsystem_bench::{
 		TestDataAvailability, TestState,
 	},
 	configuration::TestConfiguration,
-	utils::{warm_up_and_benchmark, WarmUpOptions},
+	usage::BenchmarkUsage,
 };
+use std::io::Write;
+
+const BENCH_COUNT: usize = 50;
 
 fn main() -> Result<(), String> {
 	let mut messages = vec![];
@@ -38,27 +41,33 @@ fn main() -> Result<(), String> {
 	config.num_blocks = 3;
 	config.generate_pov_sizes();
 
-	let usage = warm_up_and_benchmark(WarmUpOptions::new(&["availability-recovery"]), || {
-		let mut state = TestState::new(&config);
-		let (mut env, _protocol_config) = prepare_test(
-			config.clone(),
-			&mut state,
-			TestDataAvailability::Read(options.clone()),
-			false,
-		);
-		env.runtime().block_on(benchmark_availability_read(
-			"data_availability_read",
-			&mut env,
-			state,
-		))
-	})?;
-	println!("{}", usage);
+	let state = TestState::new(&config);
+
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> = (0..BENCH_COUNT)
+		.map(|n| {
+			print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n));
+			std::io::stdout().flush().unwrap();
+			let (mut env, _cfgs) =
+				prepare_test(&state, TestDataAvailability::Read(options.clone()), false);
+			env.runtime().block_on(benchmark_availability_read(
+				"data_availability_read",
+				&mut env,
+				&state,
+			))
+		})
+		.collect();
+	println!("\rDone!{}", " ".repeat(BENCH_COUNT));
+	let average_usage = BenchmarkUsage::average(&usages);
+	println!("{}", average_usage);
 
-	messages.extend(usage.check_network_usage(&[
-		("Received from peers", 307200.000, 0.05),
-		("Sent to peers", 1.667, 0.05),
+	// We expect no variance for received and sent
+	// but use 0.001 because we operate with floats
+	messages.extend(average_usage.check_network_usage(&[
+		("Received from peers", 307200.000, 0.001),
+		("Sent to peers", 1.667, 0.001),
 	]));
-	messages.extend(usage.check_cpu_usage(&[("availability-recovery", 11.500, 0.05)]));
+	messages.extend(average_usage.check_cpu_usage(&[("availability-recovery", 11.500, 0.05)]));
 
 	if messages.is_empty() {
 		Ok(())

diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml
@@ -56,7 +56,7 @@ bitvec = "1.0.1"
 kvdb-memorydb = "0.13.0"
 
 parity-scale-codec = { version = "3.6.1", features = ["derive", "std"] }
-tokio = "1.24.2"
+tokio = { version = "1.24.2", features = ["rt-multi-thread", "parking_lot"] }
 clap-num = "1.0.2"
 polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" }
 sp-keyring = { path = "../../../substrate/primitives/keyring" }

diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs
@@ -136,31 +136,29 @@ impl BenchCli {
 
 			let usage = match objective {
 				TestObjective::DataAvailabilityRead(opts) => {
-					let mut state = availability::TestState::new(&test_config);
+					let state = availability::TestState::new(&test_config);
 					let (mut env, _protocol_config) = availability::prepare_test(
-						test_config,
-						&mut state,
+						&state,
 						availability::TestDataAvailability::Read(opts),
 						true,
 					);
 					env.runtime().block_on(availability::benchmark_availability_read(
 						&benchmark_name,
 						&mut env,
-						state,
+						&state,
 					))
 				},
 				TestObjective::DataAvailabilityWrite => {
-					let mut state = availability::TestState::new(&test_config);
+					let state = availability::TestState::new(&test_config);
 					let (mut env, _protocol_config) = availability::prepare_test(
-						test_config,
-						&mut state,
+						&state,
 						availability::TestDataAvailability::Write,
 						true,
 					);
 					env.runtime().block_on(availability::benchmark_availability_write(
 						&benchmark_name,
 						&mut env,
-						state,
+						&state,
 					))
 				},
 				TestObjective::ApprovalVoting(ref options) => {