From f90ec6a84fe2383c43cdec1225b978d4a05c1dc1 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 16:01:50 +0100 Subject: [PATCH 01/37] Adjust av write regression tests --- ...ilability-distribution-regression-bench.rs | 17 ++++++----- .../availability-recovery-regression-bench.rs | 29 ++++++++++--------- .../node/subsystem-bench/src/lib/utils.rs | 14 +++++++-- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index bdab11298d5c..aa581d8c4fb5 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -32,17 +32,20 @@ use polkadot_subsystem_bench::{ fn main() -> Result<(), String> { let mut messages = vec![]; let mut config = TestConfiguration::default(); - // A single node effort roughly n_cores * needed_approvals / n_validators = 60 * 30 / 300 - config.n_cores = 6; + // A single node effort roughly + config.n_cores = 10; + config.n_validators = 500; config.num_blocks = 3; config.generate_pov_sizes(); let usage = warm_up_and_benchmark( - WarmUpOptions::new(&[ - "availability-distribution", - "bitfield-distribution", - "availability-store", - ]), + WarmUpOptions::new() + .subsystems(&[ + "availability-distribution", + "bitfield-distribution", + "availability-store", + ]) + .precision(0.03), || { let mut state = TestState::new(&config); let (mut env, _protocol_config) = diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index 42b1787e0450..eb2c26dcb13f 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -38,20 +38,21 @@ fn main() -> Result<(), String> { config.num_blocks = 3; config.generate_pov_sizes(); - let usage = warm_up_and_benchmark(WarmUpOptions::new(&["availability-recovery"]), || { - let mut state = TestState::new(&config); - let (mut env, _protocol_config) = prepare_test( - config.clone(), - &mut state, - TestDataAvailability::Read(options.clone()), - false, - ); - env.runtime().block_on(benchmark_availability_read( - "data_availability_read", - &mut env, - state, - )) - })?; + let usage = + warm_up_and_benchmark(WarmUpOptions::new().subsystems(&["availability-recovery"]), || { + let mut state = TestState::new(&config); + let (mut env, _protocol_config) = prepare_test( + config.clone(), + &mut state, + TestDataAvailability::Read(options.clone()), + false, + ); + env.runtime().block_on(benchmark_availability_read( + "data_availability_read", + &mut env, + state, + )) + })?; println!("{}", usage); messages.extend(usage.check_network_usage(&[ diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs index 75b72cc11b98..3e6f951c7c6f 100644 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ b/polkadot/node/subsystem-bench/src/lib/utils.rs @@ -31,8 +31,18 @@ pub struct WarmUpOptions<'a> { } impl<'a> WarmUpOptions<'a> { - pub fn new(subsystems: &'a [&'a str]) -> Self { - Self { warm_up: 100, bench: 3, precision: 0.02, subsystems } + pub fn new() -> Self { + Self { warm_up: 100, bench: 3, precision: 0.02, subsystems: &[] } + } + + pub fn subsystems(mut self, v: &'a [&'a str]) -> Self { + self.subsystems = v; + self + } + + pub fn precision(mut self, v: f64) -> Self { + self.precision = v; + self } } From be14c1f6b8887d0a4d293930b6d145d714ff218a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 16:03:52 +0100 Subject: [PATCH 02/37] Add the test --- .gitlab/pipeline/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 26e55e9385f3..2e46d3ce5f74 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -503,6 +503,7 @@ subsystem-regression-tests: - .run-immediately script: - cargo test --profile=testnet -p polkadot-availability-recovery --test availability-recovery-regression-bench --features subsystem-benchmarks + - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks tags: - benchmark allow_failure: true From 18d8401ab4fd8e52d03eac5661785c9549309840 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 16:23:34 +0100 Subject: [PATCH 03/37] Fix clippy --- polkadot/node/subsystem-bench/src/lib/utils.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs index 3e6f951c7c6f..363f45b39ecd 100644 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ b/polkadot/node/subsystem-bench/src/lib/utils.rs @@ -46,6 +46,12 @@ impl<'a> WarmUpOptions<'a> { } } +impl<'a> Default for WarmUpOptions<'a> { + fn default() -> Self { + Self::new() + } +} + pub fn warm_up_and_benchmark( options: WarmUpOptions, run: impl Fn() -> BenchmarkUsage, From f085105358577f42ecf426d806f60f5632627a37 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 16:37:37 +0100 Subject: [PATCH 04/37] Update --- .../availability-distribution-regression-bench.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index aa581d8c4fb5..1c799f8ba50f 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -60,13 +60,13 @@ fn main() -> Result<(), String> { println!("{}", usage); messages.extend(usage.check_network_usage(&[ - ("Received from peers", 443.333, 0.05), - ("Sent to peers", 21818.555, 0.05), + ("Received from peers", 433.333, 0.05), + ("Sent to peers", 18480.000, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.011, 0.05), - ("bitfield-distribution", 0.029, 0.05), - ("availability-store", 0.232, 0.05), + ("availability-distribution", 0.013, 0.05), + ("bitfield-distribution", 0.046, 0.05), + ("availability-store", 0.154, 0.05), ])); if messages.is_empty() { From 7bc1f66950728553d22370f082592b1c9acdc22f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 16:48:48 +0100 Subject: [PATCH 05/37] Update --- .../tests/availability-distribution-regression-bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 1c799f8ba50f..e812d8490f46 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -64,9 +64,9 @@ fn main() -> Result<(), String> { ("Sent to peers", 18480.000, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.013, 0.05), + ("availability-distribution", 0.012, 0.05), ("bitfield-distribution", 0.046, 0.05), - ("availability-store", 0.154, 0.05), + ("availability-store", 0.151, 0.05), ])); if messages.is_empty() { From c83802d04788db526a543ac27e43e898a575f96f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 17:51:11 +0100 Subject: [PATCH 06/37] Update options --- ...ilability-distribution-regression-bench.rs | 14 ++--- .../availability-recovery-regression-bench.rs | 2 +- .../node/subsystem-bench/src/lib/utils.rs | 56 +++++++++---------- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index e812d8490f46..7479fba3b819 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -39,13 +39,11 @@ fn main() -> Result<(), String> { config.generate_pov_sizes(); let usage = warm_up_and_benchmark( - WarmUpOptions::new() - .subsystems(&[ - "availability-distribution", - "bitfield-distribution", - "availability-store", - ]) - .precision(0.03), + WarmUpOptions::new(&[ + ("availability-distribution", 0.03), + ("bitfield-distribution", 0.02), + ("availability-store", 0.02), + ]), || { let mut state = TestState::new(&config); let (mut env, _protocol_config) = @@ -64,7 +62,7 @@ fn main() -> Result<(), String> { ("Sent to peers", 18480.000, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.012, 0.05), + ("availability-distribution", 0.012, 0.07), ("bitfield-distribution", 0.046, 0.05), ("availability-store", 0.151, 0.05), ])); diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index eb2c26dcb13f..00385f93b3f2 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -39,7 +39,7 @@ fn main() -> Result<(), String> { config.generate_pov_sizes(); let usage = - warm_up_and_benchmark(WarmUpOptions::new().subsystems(&["availability-recovery"]), || { + warm_up_and_benchmark(WarmUpOptions::new(&[("availability-recovery", 0.02)]), || { let mut state = TestState::new(&config); let (mut env, _protocol_config) = prepare_test( config.clone(), diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs index 363f45b39ecd..f7fdbaf5207a 100644 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ b/polkadot/node/subsystem-bench/src/lib/utils.rs @@ -17,38 +17,23 @@ //! Test utils use crate::usage::BenchmarkUsage; -use std::io::{stdout, Write}; +use std::{ + collections::HashMap, + io::{stdout, Write}, +}; pub struct WarmUpOptions<'a> { /// The maximum number of runs considered for marming up. pub warm_up: usize, /// The number of runs considered for benchmarking. pub bench: usize, - /// The difference in CPU usage between runs considered as normal - pub precision: f64, - /// The subsystems whose CPU usage is checked during warm-up cycles - pub subsystems: &'a [&'a str], + /// The difference in CPU usage between runs considered as normal mapped to subsystem + pub precisions: HashMap<&'a str, f64>, } impl<'a> WarmUpOptions<'a> { - pub fn new() -> Self { - Self { warm_up: 100, bench: 3, precision: 0.02, subsystems: &[] } - } - - pub fn subsystems(mut self, v: &'a [&'a str]) -> Self { - self.subsystems = v; - self - } - - pub fn precision(mut self, v: f64) -> Self { - self.precision = v; - self - } -} - -impl<'a> Default for WarmUpOptions<'a> { - fn default() -> Self { - Self::new() + pub fn new(precisions: &[(&'a str, f64)]) -> Self { + Self { warm_up: 100, bench: 3, precisions: precisions.iter().cloned().collect() } } } @@ -63,14 +48,27 @@ pub fn warm_up_and_benchmark( let curr = run(); if let Some(prev) = usages.last() { let diffs = options - .subsystems + .precisions + .keys() + .map(|&subsystem| { + curr.cpu_usage_diff(prev, subsystem) + .map(|diff| (subsystem, diff)) + .ok_or(format!("{} not found in benchmark {:?}", subsystem, prev)) + }) + .collect::, String>>()?; + let is_warmed_up = diffs .iter() - .map(|&v| { - curr.cpu_usage_diff(prev, v) - .ok_or(format!("{} not found in benchmark {:?}", v, prev)) + .map(|(subsystem, diff)| { + options + .precisions + .get(subsystem) + .map(|precision| *diff < *precision) + .ok_or(format!("{} not found in benchmark {:?}", subsystem, prev)) }) - .collect::, String>>()?; - if !diffs.iter().all(|&v| v < options.precision) { + .collect::, String>>()? + .iter() + .all(|v| *v); + if !is_warmed_up { usages.clear(); } } From e85a9d660cf9d2420327da73895ab91b5c5c364a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 18:02:59 +0100 Subject: [PATCH 07/37] Update --- .../tests/availability-distribution-regression-bench.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 7479fba3b819..848869807716 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -62,9 +62,9 @@ fn main() -> Result<(), String> { ("Sent to peers", 18480.000, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.012, 0.07), - ("bitfield-distribution", 0.046, 0.05), - ("availability-store", 0.151, 0.05), + ("availability-distribution", 0.013, 0.08), + ("bitfield-distribution", 0.050, 0.05), + ("availability-store", 0.158, 0.05), ])); if messages.is_empty() { From fca129881f4b88eba59d96af329cf93d1a3b19a9 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 18:27:47 +0100 Subject: [PATCH 08/37] Update --- .../tests/availability-distribution-regression-bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 848869807716..ecf5b5b891c1 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -62,8 +62,8 @@ fn main() -> Result<(), String> { ("Sent to peers", 18480.000, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.013, 0.08), - ("bitfield-distribution", 0.050, 0.05), + ("availability-distribution", 0.013, 0.10), + ("bitfield-distribution", 0.050, 0.08), ("availability-store", 0.158, 0.05), ])); From b70518e3d86a1b5051eaa7450b13437b096ece7d Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 14 Mar 2024 19:00:40 +0100 Subject: [PATCH 09/37] Update --- .../tests/availability-distribution-regression-bench.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index ecf5b5b891c1..247972e3f3e8 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -41,8 +41,8 @@ fn main() -> Result<(), String> { let usage = warm_up_and_benchmark( WarmUpOptions::new(&[ ("availability-distribution", 0.03), - ("bitfield-distribution", 0.02), - ("availability-store", 0.02), + ("bitfield-distribution", 0.01), + ("availability-store", 0.01), ]), || { let mut state = TestState::new(&config); @@ -64,7 +64,7 @@ fn main() -> Result<(), String> { messages.extend(usage.check_cpu_usage(&[ ("availability-distribution", 0.013, 0.10), ("bitfield-distribution", 0.050, 0.08), - ("availability-store", 0.158, 0.05), + ("availability-store", 0.158, 0.08), ])); if messages.is_empty() { From 79c90ccc5be5793b22d5e79df573cc4238eb69bf Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 15 Mar 2024 08:53:30 +0100 Subject: [PATCH 10/37] Update --- .../tests/availability-distribution-regression-bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 247972e3f3e8..c8588a542aa3 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -63,8 +63,8 @@ fn main() -> Result<(), String> { ])); messages.extend(usage.check_cpu_usage(&[ ("availability-distribution", 0.013, 0.10), - ("bitfield-distribution", 0.050, 0.08), - ("availability-store", 0.158, 0.08), + ("bitfield-distribution", 0.043, 0.10), + ("availability-store", 0.150, 0.10), ])); if messages.is_empty() { From 380742b92b29c329186d41ea33b23c7d02d33f2a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 15 Mar 2024 09:52:59 +0100 Subject: [PATCH 11/37] Update --- ...vailability-distribution-regression-bench.rs | 4 ++-- .../availability-recovery-regression-bench.rs | 2 +- polkadot/node/subsystem-bench/src/lib/utils.rs | 17 ++++++++++------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index c8588a542aa3..5eef7a296fc8 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -40,8 +40,8 @@ fn main() -> Result<(), String> { let usage = warm_up_and_benchmark( WarmUpOptions::new(&[ - ("availability-distribution", 0.03), - ("bitfield-distribution", 0.01), + ("availability-distribution", 0.06), + ("bitfield-distribution", 0.03), ("availability-store", 0.01), ]), || { diff --git a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs index 00385f93b3f2..0aafd7fe9cbf 100644 --- a/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/tests/availability-recovery-regression-bench.rs @@ -39,7 +39,7 @@ fn main() -> Result<(), String> { config.generate_pov_sizes(); let usage = - warm_up_and_benchmark(WarmUpOptions::new(&[("availability-recovery", 0.02)]), || { + warm_up_and_benchmark(WarmUpOptions::new(&[("availability-recovery", 0.01)]), || { let mut state = TestState::new(&config); let (mut env, _protocol_config) = prepare_test( config.clone(), diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs index f7fdbaf5207a..5081f79a2929 100644 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ b/polkadot/node/subsystem-bench/src/lib/utils.rs @@ -33,7 +33,7 @@ pub struct WarmUpOptions<'a> { impl<'a> WarmUpOptions<'a> { pub fn new(precisions: &[(&'a str, f64)]) -> Self { - Self { warm_up: 100, bench: 3, precisions: precisions.iter().cloned().collect() } + Self { warm_up: 10, bench: 3, precisions: precisions.iter().cloned().collect() } } } @@ -41,7 +41,8 @@ pub fn warm_up_and_benchmark( options: WarmUpOptions, run: impl Fn() -> BenchmarkUsage, ) -> Result { - println!("Warming up..."); + print!("Warming up..."); + stdout().flush().unwrap(); let mut usages = Vec::with_capacity(options.bench); for n in 1..=options.warm_up { @@ -73,17 +74,19 @@ pub fn warm_up_and_benchmark( } } usages.push(curr); - print!("\r{}%", n * 100 / options.warm_up); + print!("\rWarming up... {}%", n * 100 / options.warm_up); + stdout().flush().unwrap(); + if n == options.warm_up { + println!("\nWarm up finished"); + } if usages.len() == options.bench { - println!("\rTook {} runs to warm up", n.saturating_sub(options.bench)); + println!("\nTook {} runs to warm up", n.saturating_sub(options.bench)); break; } - stdout().flush().unwrap(); } if usages.len() != options.bench { - println!("Didn't warm up after {} runs", options.warm_up); - return Err("Can't warm up".to_string()) + return Err(format!("Didn't warm up after {} runs", options.warm_up)) } Ok(BenchmarkUsage::average(&usages)) From e024367b5ad22cefc755df97aecc886c0ab8ae29 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 15 Mar 2024 11:10:16 +0100 Subject: [PATCH 12/37] Add debug --- polkadot/node/subsystem-bench/src/lib/utils.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs index 5081f79a2929..6126ca3c7e2b 100644 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ b/polkadot/node/subsystem-bench/src/lib/utils.rs @@ -41,8 +41,7 @@ pub fn warm_up_and_benchmark( options: WarmUpOptions, run: impl Fn() -> BenchmarkUsage, ) -> Result { - print!("Warming up..."); - stdout().flush().unwrap(); + println!("Warming up..."); let mut usages = Vec::with_capacity(options.bench); for n in 1..=options.warm_up { @@ -72,13 +71,17 @@ pub fn warm_up_and_benchmark( if !is_warmed_up { usages.clear(); } + println!( + "{}/{}: {}", + n, + options.warm_up, + diffs + .iter() + .map(|(subsystem, diff)| format!("{}: {:.3}", subsystem, diff)) + .join(", ") + ); } usages.push(curr); - print!("\rWarming up... {}%", n * 100 / options.warm_up); - stdout().flush().unwrap(); - if n == options.warm_up { - println!("\nWarm up finished"); - } if usages.len() == options.bench { println!("\nTook {} runs to warm up", n.saturating_sub(options.bench)); break; From b3e1b3ea2fde91f9f095cda46c32fb9533e2314f Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 15 Mar 2024 11:17:59 +0100 Subject: [PATCH 13/37] Fix imports --- polkadot/node/subsystem-bench/src/lib/utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs index 6126ca3c7e2b..a0d73b4e90ab 100644 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ b/polkadot/node/subsystem-bench/src/lib/utils.rs @@ -17,6 +17,7 @@ //! Test utils use crate::usage::BenchmarkUsage; +use itertools::Itertools; use std::{ collections::HashMap, io::{stdout, Write}, From 320b3ef9f32ce48f764719f9b7126b06c773eeb0 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 15 Mar 2024 11:23:05 +0100 Subject: [PATCH 14/37] Fix imports --- polkadot/node/subsystem-bench/src/lib/utils.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs index a0d73b4e90ab..1334a8d04296 100644 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ b/polkadot/node/subsystem-bench/src/lib/utils.rs @@ -18,10 +18,7 @@ use crate::usage::BenchmarkUsage; use itertools::Itertools; -use std::{ - collections::HashMap, - io::{stdout, Write}, -}; +use std::collections::HashMap; pub struct WarmUpOptions<'a> { /// The maximum number of runs considered for marming up. From ed920a1d71a966fc4e987f3017092c32601ad794 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 15 Mar 2024 14:18:13 +0100 Subject: [PATCH 15/37] Update values --- .../tests/availability-distribution-regression-bench.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 5eef7a296fc8..7ad4302da1d3 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -62,9 +62,9 @@ fn main() -> Result<(), String> { ("Sent to peers", 18480.000, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.013, 0.10), - ("bitfield-distribution", 0.043, 0.10), - ("availability-store", 0.150, 0.10), + ("availability-distribution", 0.012, 0.10), + ("bitfield-distribution", 0.048, 0.10), + ("availability-store", 0.155, 0.10), ])); if messages.is_empty() { From 733ea220079272b9f2223b33539a8fafb7f6505b Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 15 Mar 2024 15:00:28 +0100 Subject: [PATCH 16/37] Increase precission --- .../tests/availability-distribution-regression-bench.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 7ad4302da1d3..9ece5cab6e84 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -62,9 +62,9 @@ fn main() -> Result<(), String> { ("Sent to peers", 18480.000, 0.05), ])); messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.012, 0.10), - ("bitfield-distribution", 0.048, 0.10), - ("availability-store", 0.155, 0.10), + ("availability-distribution", 0.012, 0.15), + ("bitfield-distribution", 0.048, 0.15), + ("availability-store", 0.155, 0.15), ])); if messages.is_empty() { From 7d596f81b64433ef0bf676feaf2d036441cb6662 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 20 Mar 2024 15:05:28 +0100 Subject: [PATCH 17/37] Move all generation out of regression tests (#3746) --- ...ilability-distribution-regression-bench.rs | 72 ++-- polkadot/node/subsystem-bench/Cargo.toml | 2 +- .../src/lib/availability/mod.rs | 312 ++++++++++-------- .../subsystem-bench/src/lib/environment.rs | 1 + polkadot/node/subsystem-bench/src/lib/lib.rs | 1 - .../subsystem-bench/src/lib/mock/av_store.rs | 1 + .../src/lib/mock/runtime_api.rs | 2 + .../node/subsystem-bench/src/lib/usage.rs | 15 +- .../node/subsystem-bench/src/lib/utils.rs | 94 ------ 9 files changed, 242 insertions(+), 258 deletions(-) delete mode 100644 polkadot/node/subsystem-bench/src/lib/utils.rs diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index 9ece5cab6e84..bbcc7269315c 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -24,10 +24,15 @@ //! - availability-store use polkadot_subsystem_bench::{ - availability::{benchmark_availability_write, prepare_test, TestDataAvailability, TestState}, + availability::{ + benchmark_availability_write, prepare_data, prepare_test, TestDataAvailability, + }, configuration::TestConfiguration, - utils::{warm_up_and_benchmark, WarmUpOptions}, + usage::BenchmarkUsage, }; +use std::io::Write; + +const BENCH_COUNT: usize = 50; fn main() -> Result<(), String> { let mut messages = vec![]; @@ -37,34 +42,55 @@ fn main() -> Result<(), String> { config.n_validators = 500; config.num_blocks = 3; config.generate_pov_sizes(); + let ( + state, + test_authorities, + block_headers, + block_infos, + chunk_fetching_requests, + signed_bitfields, + availability_state, + runtime_api, + ) = prepare_data(&config); - let usage = warm_up_and_benchmark( - WarmUpOptions::new(&[ - ("availability-distribution", 0.06), - ("bitfield-distribution", 0.03), - ("availability-store", 0.01), - ]), - || { - let mut state = TestState::new(&config); - let (mut env, _protocol_config) = - prepare_test(config.clone(), &mut state, TestDataAvailability::Write, false); + println!("Benchmarking..."); + let usages: Vec = (0..BENCH_COUNT) + .map(|n| { + print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n)); + std::io::stdout().flush().unwrap(); + let mut env = prepare_test( + &config, + &state, + &test_authorities, + &block_headers, + &availability_state, + &runtime_api, + TestDataAvailability::Write, + false, + ); env.runtime().block_on(benchmark_availability_write( "data_availability_write", &mut env, - state, + &state, + &block_infos, + &chunk_fetching_requests, + &signed_bitfields, )) - }, - )?; - println!("{}", usage); + }) + .collect(); + println!("\rDone!{}", " ".repeat(BENCH_COUNT)); + let average_usage = BenchmarkUsage::average(&usages); + println!("{}", average_usage); - messages.extend(usage.check_network_usage(&[ - ("Received from peers", 433.333, 0.05), - ("Sent to peers", 18480.000, 0.05), + messages.extend(average_usage.check_network_usage(&[ + ("Received from peers", 433.3, 0.05), + ("Sent to peers", 18480.0, 0.05), ])); - messages.extend(usage.check_cpu_usage(&[ - ("availability-distribution", 0.012, 0.15), - ("bitfield-distribution", 0.048, 0.15), - ("availability-store", 0.155, 0.15), + + messages.extend(average_usage.check_cpu_usage(&[ + ("availability-distribution", 0.011, 0.05), + ("bitfield-distribution", 0.050, 0.05), + ("availability-store", 0.159, 0.05), ])); if messages.is_empty() { diff --git a/polkadot/node/subsystem-bench/Cargo.toml b/polkadot/node/subsystem-bench/Cargo.toml index 71711ad0fbd0..2570fe9cfa23 100644 --- a/polkadot/node/subsystem-bench/Cargo.toml +++ b/polkadot/node/subsystem-bench/Cargo.toml @@ -56,7 +56,7 @@ bitvec = "1.0.1" kvdb-memorydb = "0.13.0" parity-scale-codec = { version = "3.6.1", features = ["derive", "std"] } -tokio = "1.24.2" +tokio = { version = "1.24.2", features = ["rt-multi-thread", "parking_lot"] } clap-num = "1.0.2" polkadot-node-subsystem-test-helpers = { path = "../subsystem-test-helpers" } sp-keyring = { path = "../../../substrate/primitives/keyring" } diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index dc4e1e403102..3f5d07798bdf 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -15,7 +15,7 @@ // along with Polkadot. If not, see . use crate::{ - configuration::TestConfiguration, + configuration::{TestAuthorities, TestConfiguration}, dummy_builder, environment::{TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, mock::{ @@ -58,16 +58,13 @@ use polkadot_node_subsystem_types::{ messages::{AvailabilityStoreMessage, NetworkBridgeEvent}, Span, }; -use polkadot_overseer::{metrics::Metrics as OverseerMetrics, Handle as OverseerHandle}; +use polkadot_overseer::{metrics::Metrics as OverseerMetrics, BlockInfo, Handle as OverseerHandle}; use polkadot_primitives::{ AvailabilityBitfield, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, Header, PersistedValidationData, Signed, SigningContext, ValidatorIndex, }; use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; -use sc_network::{ - request_responses::{IncomingRequest as RawIncomingRequest, ProtocolConfig}, - PeerId, -}; +use sc_network::request_responses::IncomingRequest as RawIncomingRequest; use sc_service::SpawnTaskHandle; use serde::{Deserialize, Serialize}; use sp_core::H256; @@ -149,32 +146,21 @@ fn build_overseer_for_availability_write( (overseer, OverseerHandle::new(raw_handle)) } -/// Takes a test configuration and uses it to create the `TestEnvironment`. -pub fn prepare_test( - config: TestConfiguration, - state: &mut TestState, - mode: TestDataAvailability, - with_prometheus_endpoint: bool, -) -> (TestEnvironment, Vec) { - prepare_test_inner( - config, - state, - mode, - TestEnvironmentDependencies::default(), - with_prometheus_endpoint, - ) -} - -fn prepare_test_inner( - config: TestConfiguration, - state: &mut TestState, - mode: TestDataAvailability, - dependencies: TestEnvironmentDependencies, - with_prometheus_endpoint: bool, -) -> (TestEnvironment, Vec) { +pub fn prepare_data( + config: &TestConfiguration, +) -> ( + TestState, + TestAuthorities, + HashMap, + Vec, + Vec>>, + HashMap>, + NetworkAvailabilityState, + MockRuntimeApi, +) { + let mut state = TestState::new(config); // Generate test authorities. let test_authorities = config.generate_authorities(); - let mut candidate_hashes: HashMap> = HashMap::new(); // Prepare per block candidates. @@ -213,21 +199,117 @@ fn prepare_test_inner( chunks: state.chunks.clone(), }; - let mut req_cfgs = Vec::new(); + let block_headers = (1..=config.num_blocks) + .map(|block_number| { + ( + Hash::repeat_byte(block_number as u8), + Header { + digest: Default::default(), + number: block_number as BlockNumber, + parent_hash: Default::default(), + extrinsics_root: Default::default(), + state_root: Default::default(), + }, + ) + }) + .collect::>(); + + let block_infos: Vec = (1..=config.num_blocks) + .map(|block_num| { + let relay_block_hash = Hash::repeat_byte(block_num as u8); + new_block_import_info(relay_block_hash, block_num as BlockNumber) + }) + .collect(); + + let chunk_fetching_requests = state + .backed_candidates() + .iter() + .map(|candidate| { + (0..config.n_validators) + .map(|index| { + ChunkFetchingRequest { + candidate_hash: candidate.hash(), + index: ValidatorIndex(index as u32), + } + .encode() + }) + .collect::>() + }) + .collect::>(); + + let signed_bitfields: HashMap> = block_infos + .iter() + .map(|block_info| { + let signing_context = SigningContext { session_index: 0, parent_hash: block_info.hash }; + let messages = (0..config.n_validators) + .map(|index| { + let validator_public = test_authorities + .validator_public + .get(index) + .expect("All validator keys are known"); + + // Node has all the chunks in the world. + let payload: AvailabilityBitfield = + AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); + let signed_bitfield = Signed::::sign( + &test_authorities.keyring.keystore(), + payload, + &signing_context, + ValidatorIndex(index as u32), + validator_public, + ) + .ok() + .flatten() + .expect("should be signed"); + + peer_bitfield_message_v2(block_info.hash, signed_bitfield) + }) + .collect::>(); + + (block_info.hash, messages) + }) + .collect(); + + ( + state, + test_authorities, + block_headers, + block_infos, + chunk_fetching_requests, + signed_bitfields, + availability_state, + runtime_api, + ) +} + +/// Takes a test configuration and uses it to create the `TestEnvironment`. +pub fn prepare_test( + config: &TestConfiguration, + state: &TestState, + test_authorities: &TestAuthorities, + block_headers: &HashMap, + availability_state: &NetworkAvailabilityState, + runtime_api: &MockRuntimeApi, + mode: TestDataAvailability, + with_prometheus_endpoint: bool, +) -> TestEnvironment { + let dependencies = TestEnvironmentDependencies::default(); - let (collation_req_receiver, collation_req_cfg) = + let (collation_req_receiver, _collation_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - req_cfgs.push(collation_req_cfg); - let (pov_req_receiver, pov_req_cfg) = + let (pov_req_receiver, _pov_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); let (chunk_req_receiver, chunk_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - req_cfgs.push(pov_req_cfg); - let (network, network_interface, network_receiver) = - new_network(&config, &dependencies, &test_authorities, vec![Arc::new(availability_state)]); + let (network, network_interface, network_receiver) = new_network( + &config, + &dependencies, + &test_authorities, + vec![Arc::new(availability_state.clone())], + ); let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( network.clone(), @@ -262,7 +344,7 @@ fn prepare_test_inner( build_overseer_for_availability_read( dependencies.task_manager.spawn_handle(), - runtime_api, + runtime_api.clone(), av_store, (network_bridge_tx, network_bridge_rx), subsystem, @@ -276,28 +358,13 @@ fn prepare_test_inner( Metrics::try_register(&dependencies.registry).unwrap(), ); - let block_headers = (1..=config.num_blocks) - .map(|block_number| { - ( - Hash::repeat_byte(block_number as u8), - Header { - digest: Default::default(), - number: block_number as BlockNumber, - parent_hash: Default::default(), - extrinsics_root: Default::default(), - state_root: Default::default(), - }, - ) - }) - .collect::>(); - - let chain_api_state = ChainApiState { block_headers }; + let chain_api_state = ChainApiState { block_headers: block_headers.clone() }; let chain_api = MockChainApi::new(chain_api_state); let bitfield_distribution = BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); build_overseer_for_availability_write( dependencies.task_manager.spawn_handle(), - runtime_api, + runtime_api.clone(), (network_bridge_tx, network_bridge_rx), availability_distribution, chain_api, @@ -308,17 +375,14 @@ fn prepare_test_inner( }, }; - ( - TestEnvironment::new( - dependencies, - config, - network, - overseer, - overseer_handle, - test_authorities, - with_prometheus_endpoint, - ), - req_cfgs, + TestEnvironment::new( + dependencies, + config.clone(), + network, + overseer, + overseer_handle, + test_authorities.clone(), + with_prometheus_endpoint, ) } @@ -443,8 +507,8 @@ impl TestState { _self } - pub fn backed_candidates(&mut self) -> &mut Vec { - &mut self.backed_candidates + pub fn backed_candidates(&self) -> &Vec { + &self.backed_candidates } } @@ -519,7 +583,10 @@ pub async fn benchmark_availability_read( pub async fn benchmark_availability_write( benchmark_name: &str, env: &mut TestEnvironment, - mut state: TestState, + state: &TestState, + block_infos: &[BlockInfo], + chunk_fetching_requests: &[Vec>], + signed_bitfields: &HashMap>, ) -> BenchmarkUsage { let config = env.config().clone(); @@ -550,15 +617,14 @@ pub async fn benchmark_availability_write( gum::info!(target: LOG_TARGET, "Done"); let test_start = Instant::now(); - - for block_num in 1..=env.config().num_blocks { + for block_info in block_infos { + let block_num = block_info.number as usize; gum::info!(target: LOG_TARGET, "Current block #{}", block_num); env.metrics().set_current_block(block_num); let block_start_ts = Instant::now(); - let relay_block_hash = Hash::repeat_byte(block_num as u8); - env.import_block(new_block_import_info(relay_block_hash, block_num as BlockNumber)) - .await; + let relay_block_hash = block_info.hash.clone(); + env.import_block(block_info.clone()).await; // Inform bitfield distribution about our view of current test block let message = polkadot_node_subsystem_types::messages::BitfieldDistributionMessage::NetworkBridgeUpdate( @@ -569,82 +635,58 @@ pub async fn benchmark_availability_write( let chunk_fetch_start_ts = Instant::now(); // Request chunks of our own backed candidate from all other validators. - let mut receivers = Vec::new(); - for index in 1..config.n_validators { - let (pending_response, pending_response_receiver) = oneshot::channel(); - - let request = RawIncomingRequest { - peer: PeerId::random(), - payload: ChunkFetchingRequest { - candidate_hash: state.backed_candidates()[block_num - 1].hash(), - index: ValidatorIndex(index as u32), + let payloads = chunk_fetching_requests.get(block_num - 1).expect("pregenerated"); + let receivers = (1..config.n_validators) + .map(|index| { + let (pending_response, pending_response_receiver) = oneshot::channel(); + + let peer_id = + *env.authorities().peer_ids.get(index).expect("all validators have ids"); + let payload = payloads.get(index).expect("pregenerated").clone(); + let request = RawIncomingRequest { peer: peer_id, payload, pending_response }; + let peer = env + .authorities() + .validator_authority_id + .get(index) + .expect("all validators have keys"); + + if env.network().is_peer_connected(peer) && + env.network().send_request_from_peer(peer, request).is_ok() + { + Some(pending_response_receiver) + } else { + None } - .encode(), - pending_response, - }; - - let peer = env - .authorities() - .validator_authority_id - .get(index) - .expect("all validators have keys"); - - if env.network().is_peer_connected(peer) && - env.network().send_request_from_peer(peer, request).is_ok() - { - receivers.push(pending_response_receiver); - } - } + }) + .filter_map(|v| v); gum::info!(target: LOG_TARGET, "Waiting for all emulated peers to receive their chunk from us ..."); - for receiver in receivers.into_iter() { - let response = receiver.await.expect("Chunk is always served successfully"); - // TODO: check if chunk is the one the peer expects to receive. - assert!(response.result.is_ok()); - } - let chunk_fetch_duration = Instant::now().sub(chunk_fetch_start_ts).as_millis(); + let responses = futures::future::try_join_all(receivers) + .await + .expect("Chunk is always served successfully"); + // TODO: check if chunk is the one the peer expects to receive. + assert!(responses.iter().all(|v| v.result.is_ok())); + let chunk_fetch_duration = Instant::now().sub(chunk_fetch_start_ts).as_millis(); gum::info!(target: LOG_TARGET, "All chunks received in {}ms", chunk_fetch_duration); - let signing_context = SigningContext { session_index: 0, parent_hash: relay_block_hash }; let network = env.network().clone(); let authorities = env.authorities().clone(); - let n_validators = config.n_validators; // Spawn a task that will generate `n_validator` - 1 signed bitfiends and // send them from the emulated peers to the subsystem. // TODO: Implement topology. - env.spawn_blocking("send-bitfields", async move { - for index in 1..n_validators { - let validator_public = - authorities.validator_public.get(index).expect("All validator keys are known"); - - // Node has all the chunks in the world. - let payload: AvailabilityBitfield = - AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); - // TODO(soon): Use pre-signed messages. This is quite intensive on the CPU. - let signed_bitfield = Signed::::sign( - &authorities.keyring.keystore(), - payload, - &signing_context, - ValidatorIndex(index as u32), - validator_public, - ) - .ok() - .flatten() - .expect("should be signed"); - - let from_peer = &authorities.validator_authority_id[index]; - - let message = peer_bitfield_message_v2(relay_block_hash, signed_bitfield); + let messages = signed_bitfields.get(&relay_block_hash).expect("pregenerated").clone(); + for index in 1..config.n_validators { + let from_peer = &authorities.validator_authority_id[index]; + let message = messages.get(index).expect("pregenerated").clone(); - // Send the action from peer only if it is connected to our node. - if network.is_peer_connected(from_peer) { - let _ = network.send_message_from_peer(from_peer, message); - } + // Send the action from peer only if it is connected to our node. + if network.is_peer_connected(from_peer) { + let _ = network.send_message_from_peer(from_peer, message); } - }); + } gum::info!( "Waiting for {} bitfields to be received and processed", diff --git a/polkadot/node/subsystem-bench/src/lib/environment.rs b/polkadot/node/subsystem-bench/src/lib/environment.rs index 958ed50d0894..2d80d75a14aa 100644 --- a/polkadot/node/subsystem-bench/src/lib/environment.rs +++ b/polkadot/node/subsystem-bench/src/lib/environment.rs @@ -118,6 +118,7 @@ fn new_runtime() -> tokio::runtime::Runtime { .thread_name("subsystem-bench") .enable_all() .thread_stack_size(3 * 1024 * 1024) + .worker_threads(4) .build() .unwrap() } diff --git a/polkadot/node/subsystem-bench/src/lib/lib.rs b/polkadot/node/subsystem-bench/src/lib/lib.rs index ef2724abc989..d06f2822a895 100644 --- a/polkadot/node/subsystem-bench/src/lib/lib.rs +++ b/polkadot/node/subsystem-bench/src/lib/lib.rs @@ -26,4 +26,3 @@ pub(crate) mod keyring; pub(crate) mod mock; pub(crate) mod network; pub mod usage; -pub mod utils; diff --git a/polkadot/node/subsystem-bench/src/lib/mock/av_store.rs b/polkadot/node/subsystem-bench/src/lib/mock/av_store.rs index 9064f17940f0..080644da92a0 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/av_store.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/av_store.rs @@ -41,6 +41,7 @@ const LOG_TARGET: &str = "subsystem-bench::av-store-mock"; /// Mockup helper. Contains Ccunks and full availability data of all parachain blocks /// used in a test. +#[derive(Clone)] pub struct NetworkAvailabilityState { pub candidate_hashes: HashMap, pub available_data: Vec, diff --git a/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs index 53faf562f03c..3c39de870a28 100644 --- a/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs +++ b/polkadot/node/subsystem-bench/src/lib/mock/runtime_api.rs @@ -36,6 +36,7 @@ use std::collections::HashMap; const LOG_TARGET: &str = "subsystem-bench::runtime-api-mock"; /// Minimal state to answer requests. +#[derive(Clone)] pub struct RuntimeApiState { // All authorities in the test, authorities: TestAuthorities, @@ -49,6 +50,7 @@ pub struct RuntimeApiState { } /// A mocked `runtime-api` subsystem. +#[derive(Clone)] pub struct MockRuntimeApi { state: RuntimeApiState, config: TestConfiguration, diff --git a/polkadot/node/subsystem-bench/src/lib/usage.rs b/polkadot/node/subsystem-bench/src/lib/usage.rs index ef60d67372ae..7172969a8f92 100644 --- a/polkadot/node/subsystem-bench/src/lib/usage.rs +++ b/polkadot/node/subsystem-bench/src/lib/usage.rs @@ -17,6 +17,7 @@ //! Test usage implementation use colored::Colorize; +use itertools::Itertools; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -37,10 +38,16 @@ impl std::fmt::Display for BenchmarkUsage { self.network_usage .iter() .map(|v| v.to_string()) + .sorted() .collect::>() .join("\n"), format!("{:<32}{:>12}{:>12}", "CPU usage, seconds", "total", "per block").blue(), - self.cpu_usage.iter().map(|v| v.to_string()).collect::>().join("\n") + self.cpu_usage + .iter() + .map(|v| v.to_string()) + .sorted() + .collect::>() + .join("\n") ) } } @@ -101,8 +108,8 @@ fn check_resource_usage( None } else { Some(format!( - "The resource `{}` is expected to be equal to {} with a precision {}, but the current value is {}", - resource_name, base, precision, usage.per_block + "The resource `{}` is expected to be equal to {} with a precision {}, but the current value is {} ({})", + resource_name, base, precision, usage.per_block, diff )) } } else { @@ -119,7 +126,7 @@ pub struct ResourceUsage { impl std::fmt::Display for ResourceUsage { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "{:<32}{:>12.3}{:>12.3}", self.resource_name.cyan(), self.total, self.per_block) + write!(f, "{:<32}{:>12.4}{:>12.4}", self.resource_name.cyan(), self.total, self.per_block) } } diff --git a/polkadot/node/subsystem-bench/src/lib/utils.rs b/polkadot/node/subsystem-bench/src/lib/utils.rs deleted file mode 100644 index 1334a8d04296..000000000000 --- a/polkadot/node/subsystem-bench/src/lib/utils.rs +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (C) Parity Technologies (UK) Ltd. -// This file is part of Polkadot. - -// Polkadot is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Polkadot is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Polkadot. If not, see . - -//! Test utils - -use crate::usage::BenchmarkUsage; -use itertools::Itertools; -use std::collections::HashMap; - -pub struct WarmUpOptions<'a> { - /// The maximum number of runs considered for marming up. - pub warm_up: usize, - /// The number of runs considered for benchmarking. - pub bench: usize, - /// The difference in CPU usage between runs considered as normal mapped to subsystem - pub precisions: HashMap<&'a str, f64>, -} - -impl<'a> WarmUpOptions<'a> { - pub fn new(precisions: &[(&'a str, f64)]) -> Self { - Self { warm_up: 10, bench: 3, precisions: precisions.iter().cloned().collect() } - } -} - -pub fn warm_up_and_benchmark( - options: WarmUpOptions, - run: impl Fn() -> BenchmarkUsage, -) -> Result { - println!("Warming up..."); - let mut usages = Vec::with_capacity(options.bench); - - for n in 1..=options.warm_up { - let curr = run(); - if let Some(prev) = usages.last() { - let diffs = options - .precisions - .keys() - .map(|&subsystem| { - curr.cpu_usage_diff(prev, subsystem) - .map(|diff| (subsystem, diff)) - .ok_or(format!("{} not found in benchmark {:?}", subsystem, prev)) - }) - .collect::, String>>()?; - let is_warmed_up = diffs - .iter() - .map(|(subsystem, diff)| { - options - .precisions - .get(subsystem) - .map(|precision| *diff < *precision) - .ok_or(format!("{} not found in benchmark {:?}", subsystem, prev)) - }) - .collect::, String>>()? - .iter() - .all(|v| *v); - if !is_warmed_up { - usages.clear(); - } - println!( - "{}/{}: {}", - n, - options.warm_up, - diffs - .iter() - .map(|(subsystem, diff)| format!("{}: {:.3}", subsystem, diff)) - .join(", ") - ); - } - usages.push(curr); - if usages.len() == options.bench { - println!("\nTook {} runs to warm up", n.saturating_sub(options.bench)); - break; - } - } - - if usages.len() != options.bench { - return Err(format!("Didn't warm up after {} runs", options.warm_up)) - } - - Ok(BenchmarkUsage::average(&usages)) -} From c1d831a210515479c640761b1a6da72ae5965001 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 20 Mar 2024 15:27:08 +0100 Subject: [PATCH 18/37] Add fields to state --- ...ilability-distribution-regression-bench.rs | 15 +------ .../src/lib/availability/mod.rs | 41 +++++++------------ 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index bbcc7269315c..d0e037ed1c7d 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -42,16 +42,8 @@ fn main() -> Result<(), String> { config.n_validators = 500; config.num_blocks = 3; config.generate_pov_sizes(); - let ( - state, - test_authorities, - block_headers, - block_infos, - chunk_fetching_requests, - signed_bitfields, - availability_state, - runtime_api, - ) = prepare_data(&config); + let (state, test_authorities, block_headers, availability_state, runtime_api) = + prepare_data(&config); println!("Benchmarking..."); let usages: Vec = (0..BENCH_COUNT) @@ -72,9 +64,6 @@ fn main() -> Result<(), String> { "data_availability_write", &mut env, &state, - &block_infos, - &chunk_fetching_requests, - &signed_bitfields, )) }) .collect(); diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 3f5d07798bdf..6108eb57d1cf 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -148,16 +148,7 @@ fn build_overseer_for_availability_write( pub fn prepare_data( config: &TestConfiguration, -) -> ( - TestState, - TestAuthorities, - HashMap, - Vec, - Vec>>, - HashMap>, - NetworkAvailabilityState, - MockRuntimeApi, -) { +) -> (TestState, TestAuthorities, HashMap, NetworkAvailabilityState, MockRuntimeApi) { let mut state = TestState::new(config); // Generate test authorities. let test_authorities = config.generate_authorities(); @@ -270,16 +261,11 @@ pub fn prepare_data( }) .collect(); - ( - state, - test_authorities, - block_headers, - block_infos, - chunk_fetching_requests, - signed_bitfields, - availability_state, - runtime_api, - ) + state.block_infos = block_infos; + state.chunk_fetching_requests = chunk_fetching_requests; + state.signed_bitfields = signed_bitfields; + + (state, test_authorities, block_headers, availability_state, runtime_api) } /// Takes a test configuration and uses it to create the `TestEnvironment`. @@ -407,6 +393,9 @@ pub struct TestState { chunks: Vec>, // Per relay chain block - candidate backed by our backing group backed_candidates: Vec, + block_infos: Vec, + chunk_fetching_requests: Vec>>, + signed_bitfields: HashMap>, } impl TestState { @@ -501,6 +490,9 @@ impl TestState { candidates: Vec::new().into_iter().cycle(), backed_candidates: Vec::new(), config, + block_infos: Default::default(), + chunk_fetching_requests: Default::default(), + signed_bitfields: Default::default(), }; _self.generate_candidates(); @@ -584,9 +576,6 @@ pub async fn benchmark_availability_write( benchmark_name: &str, env: &mut TestEnvironment, state: &TestState, - block_infos: &[BlockInfo], - chunk_fetching_requests: &[Vec>], - signed_bitfields: &HashMap>, ) -> BenchmarkUsage { let config = env.config().clone(); @@ -617,7 +606,7 @@ pub async fn benchmark_availability_write( gum::info!(target: LOG_TARGET, "Done"); let test_start = Instant::now(); - for block_info in block_infos { + for block_info in state.block_infos.iter() { let block_num = block_info.number as usize; gum::info!(target: LOG_TARGET, "Current block #{}", block_num); env.metrics().set_current_block(block_num); @@ -635,7 +624,7 @@ pub async fn benchmark_availability_write( let chunk_fetch_start_ts = Instant::now(); // Request chunks of our own backed candidate from all other validators. - let payloads = chunk_fetching_requests.get(block_num - 1).expect("pregenerated"); + let payloads = state.chunk_fetching_requests.get(block_num - 1).expect("pregenerated"); let receivers = (1..config.n_validators) .map(|index| { let (pending_response, pending_response_receiver) = oneshot::channel(); @@ -677,7 +666,7 @@ pub async fn benchmark_availability_write( // Spawn a task that will generate `n_validator` - 1 signed bitfiends and // send them from the emulated peers to the subsystem. // TODO: Implement topology. - let messages = signed_bitfields.get(&relay_block_hash).expect("pregenerated").clone(); + let messages = state.signed_bitfields.get(&relay_block_hash).expect("pregenerated").clone(); for index in 1..config.n_validators { let from_peer = &authorities.validator_authority_id[index]; let message = messages.get(index).expect("pregenerated").clone(); From b108fc21daa542f82ee1c2c544b0707088546af5 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 20 Mar 2024 16:40:39 +0100 Subject: [PATCH 19/37] Update --- ...ilability-distribution-regression-bench.rs | 18 +- .../src/lib/availability/mod.rs | 478 ++++++++++-------- 2 files changed, 257 insertions(+), 239 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index d0e037ed1c7d..f209b9c27acd 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -24,9 +24,7 @@ //! - availability-store use polkadot_subsystem_bench::{ - availability::{ - benchmark_availability_write, prepare_data, prepare_test, TestDataAvailability, - }, + availability::{benchmark_availability_write, prepare_availability_write_test, TestState}, configuration::TestConfiguration, usage::BenchmarkUsage, }; @@ -42,24 +40,14 @@ fn main() -> Result<(), String> { config.n_validators = 500; config.num_blocks = 3; config.generate_pov_sizes(); - let (state, test_authorities, block_headers, availability_state, runtime_api) = - prepare_data(&config); + let state = TestState::new(&config); println!("Benchmarking..."); let usages: Vec = (0..BENCH_COUNT) .map(|n| { print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n)); std::io::stdout().flush().unwrap(); - let mut env = prepare_test( - &config, - &state, - &test_authorities, - &block_headers, - &availability_state, - &runtime_api, - TestDataAvailability::Write, - false, - ); + let mut env = prepare_availability_write_test(&state, false); env.runtime().block_on(benchmark_availability_write( "data_availability_write", &mut env, diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 6108eb57d1cf..998d82152bea 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -146,228 +146,149 @@ fn build_overseer_for_availability_write( (overseer, OverseerHandle::new(raw_handle)) } -pub fn prepare_data( - config: &TestConfiguration, -) -> (TestState, TestAuthorities, HashMap, NetworkAvailabilityState, MockRuntimeApi) { - let mut state = TestState::new(config); - // Generate test authorities. - let test_authorities = config.generate_authorities(); - let mut candidate_hashes: HashMap> = HashMap::new(); - - // Prepare per block candidates. - // Genesis block is always finalized, so we start at 1. - for block_num in 1..=config.num_blocks { - for _ in 0..config.n_cores { - candidate_hashes - .entry(Hash::repeat_byte(block_num as u8)) - .or_default() - .push(state.next_candidate().expect("Cycle iterator")) - } +/// Takes a test configuration and uses it to create the `TestEnvironment`. +pub fn prepare_availability_read_test( + state: &TestState, + test_authorities: &TestAuthorities, + options: &DataAvailabilityReadOptions, + with_prometheus_endpoint: bool, +) -> TestEnvironment { + let (collation_req_receiver, _) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + let (_, chunk_req_cfg) = IncomingRequest::::get_config_receiver( + &ReqProtocolNames::new(GENESIS_HASH, None), + ); - // First candidate is our backed candidate. - state.backed_candidates.push( - candidate_hashes - .get(&Hash::repeat_byte(block_num as u8)) - .expect("just inserted above") - .first() - .expect("just inserted above") - .clone(), - ); - } + let dependencies = TestEnvironmentDependencies::default(); + let availability_state = NetworkAvailabilityState { + candidate_hashes: state.candidate_hashes.clone(), + available_data: state.available_data.clone(), + chunks: state.chunks.clone(), + }; + let (network, network_interface, network_receiver) = new_network( + &state.config, + &dependencies, + &test_authorities, + vec![Arc::new(availability_state.clone())], + ); + let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( + network.clone(), + network_interface.subsystem_sender(), + test_authorities.clone(), + ); + let network_bridge_rx = + network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); + let subsystem = if options.fetch_from_backers { + AvailabilityRecoverySubsystem::with_fast_path( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + } else { + AvailabilityRecoverySubsystem::with_chunks_only( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + }; + + // Use a mocked av-store. + let av_store = + av_store::MockAvailabilityStore::new(state.chunks.clone(), state.candidate_hashes.clone()); let runtime_api = runtime_api::MockRuntimeApi::new( - config.clone(), - test_authorities.clone(), - candidate_hashes, + state.config.clone(), + state.test_authorities.clone(), + state.candidate_hashes_2.clone(), Default::default(), Default::default(), 0, ); - let availability_state = NetworkAvailabilityState { - candidate_hashes: state.candidate_hashes.clone(), - available_data: state.available_data.clone(), - chunks: state.chunks.clone(), - }; + let (overseer, overseer_handle) = build_overseer_for_availability_read( + dependencies.task_manager.spawn_handle(), + runtime_api, + av_store, + (network_bridge_tx, network_bridge_rx), + subsystem, + &dependencies, + ); - let block_headers = (1..=config.num_blocks) - .map(|block_number| { - ( - Hash::repeat_byte(block_number as u8), - Header { - digest: Default::default(), - number: block_number as BlockNumber, - parent_hash: Default::default(), - extrinsics_root: Default::default(), - state_root: Default::default(), - }, - ) - }) - .collect::>(); - - let block_infos: Vec = (1..=config.num_blocks) - .map(|block_num| { - let relay_block_hash = Hash::repeat_byte(block_num as u8); - new_block_import_info(relay_block_hash, block_num as BlockNumber) - }) - .collect(); - - let chunk_fetching_requests = state - .backed_candidates() - .iter() - .map(|candidate| { - (0..config.n_validators) - .map(|index| { - ChunkFetchingRequest { - candidate_hash: candidate.hash(), - index: ValidatorIndex(index as u32), - } - .encode() - }) - .collect::>() - }) - .collect::>(); - - let signed_bitfields: HashMap> = block_infos - .iter() - .map(|block_info| { - let signing_context = SigningContext { session_index: 0, parent_hash: block_info.hash }; - let messages = (0..config.n_validators) - .map(|index| { - let validator_public = test_authorities - .validator_public - .get(index) - .expect("All validator keys are known"); - - // Node has all the chunks in the world. - let payload: AvailabilityBitfield = - AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); - let signed_bitfield = Signed::::sign( - &test_authorities.keyring.keystore(), - payload, - &signing_context, - ValidatorIndex(index as u32), - validator_public, - ) - .ok() - .flatten() - .expect("should be signed"); - - peer_bitfield_message_v2(block_info.hash, signed_bitfield) - }) - .collect::>(); - - (block_info.hash, messages) - }) - .collect(); - - state.block_infos = block_infos; - state.chunk_fetching_requests = chunk_fetching_requests; - state.signed_bitfields = signed_bitfields; - - (state, test_authorities, block_headers, availability_state, runtime_api) + TestEnvironment::new( + dependencies, + state.config.clone(), + network, + overseer, + overseer_handle, + test_authorities.clone(), + with_prometheus_endpoint, + ) } /// Takes a test configuration and uses it to create the `TestEnvironment`. -pub fn prepare_test( - config: &TestConfiguration, +pub fn prepare_availability_write_test( state: &TestState, - test_authorities: &TestAuthorities, - block_headers: &HashMap, - availability_state: &NetworkAvailabilityState, - runtime_api: &MockRuntimeApi, - mode: TestDataAvailability, with_prometheus_endpoint: bool, ) -> TestEnvironment { - let dependencies = TestEnvironmentDependencies::default(); - - let (collation_req_receiver, _collation_req_cfg) = + let (pov_req_receiver, _) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - - let (pov_req_receiver, _pov_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - let (chunk_req_receiver, chunk_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + let dependencies = TestEnvironmentDependencies::default(); + let availability_state = NetworkAvailabilityState { + candidate_hashes: state.candidate_hashes.clone(), + available_data: state.available_data.clone(), + chunks: state.chunks.clone(), + }; let (network, network_interface, network_receiver) = new_network( - &config, + &state.config, &dependencies, - &test_authorities, + &state.test_authorities, vec![Arc::new(availability_state.clone())], ); - let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( network.clone(), network_interface.subsystem_sender(), - test_authorities.clone(), + state.test_authorities.clone(), ); - let network_bridge_rx = network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); - let (overseer, overseer_handle) = match &mode { - TestDataAvailability::Read(options) => { - let use_fast_path = options.fetch_from_backers; - - let subsystem = if use_fast_path { - AvailabilityRecoverySubsystem::with_fast_path( - collation_req_receiver, - Metrics::try_register(&dependencies.registry).unwrap(), - ) - } else { - AvailabilityRecoverySubsystem::with_chunks_only( - collation_req_receiver, - Metrics::try_register(&dependencies.registry).unwrap(), - ) - }; - - // Use a mocked av-store. - let av_store = av_store::MockAvailabilityStore::new( - state.chunks.clone(), - state.candidate_hashes.clone(), - ); - - build_overseer_for_availability_read( - dependencies.task_manager.spawn_handle(), - runtime_api.clone(), - av_store, - (network_bridge_tx, network_bridge_rx), - subsystem, - &dependencies, - ) - }, - TestDataAvailability::Write => { - let availability_distribution = AvailabilityDistributionSubsystem::new( - test_authorities.keyring.keystore(), - IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, - Metrics::try_register(&dependencies.registry).unwrap(), - ); + let availability_distribution = AvailabilityDistributionSubsystem::new( + state.test_authorities.keyring.keystore(), + IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, + Metrics::try_register(&dependencies.registry).unwrap(), + ); - let chain_api_state = ChainApiState { block_headers: block_headers.clone() }; - let chain_api = MockChainApi::new(chain_api_state); - let bitfield_distribution = - BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); - build_overseer_for_availability_write( - dependencies.task_manager.spawn_handle(), - runtime_api.clone(), - (network_bridge_tx, network_bridge_rx), - availability_distribution, - chain_api, - new_av_store(&dependencies), - bitfield_distribution, - &dependencies, - ) - }, - }; + let chain_api_state = ChainApiState { block_headers: state.block_headers.clone() }; + let chain_api = MockChainApi::new(chain_api_state); + let bitfield_distribution = + BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); + let runtime_api = runtime_api::MockRuntimeApi::new( + state.config.clone(), + state.test_authorities.clone(), + state.candidate_hashes_2.clone(), + Default::default(), + Default::default(), + 0, + ); + let (overseer, overseer_handle) = build_overseer_for_availability_write( + dependencies.task_manager.spawn_handle(), + runtime_api, + (network_bridge_tx, network_bridge_rx), + availability_distribution, + chain_api, + new_av_store(&dependencies), + bitfield_distribution, + &dependencies, + ); TestEnvironment::new( dependencies, - config.clone(), + state.config.clone(), network, overseer, overseer_handle, - test_authorities.clone(), + state.test_authorities.clone(), with_prometheus_endpoint, ) } @@ -396,48 +317,15 @@ pub struct TestState { block_infos: Vec, chunk_fetching_requests: Vec>>, signed_bitfields: HashMap>, + block_headers: HashMap, + test_authorities: TestAuthorities, + candidate_hashes_2: HashMap>, } impl TestState { - pub fn next_candidate(&mut self) -> Option { - let candidate = self.candidates.next(); - let candidate_hash = candidate.as_ref().unwrap().hash(); - gum::trace!(target: LOG_TARGET, "Next candidate selected {:?}", candidate_hash); - candidate - } - - /// Generate candidates to be used in the test. - fn generate_candidates(&mut self) { - let count = self.config.n_cores * self.config.num_blocks; - gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); - - // Generate all candidates - self.candidates = (0..count) - .map(|index| { - let pov_size = self.pov_sizes.next().expect("This is a cycle; qed"); - let candidate_index = *self - .pov_size_to_candidate - .get(&pov_size) - .expect("pov_size always exists; qed"); - let mut candidate_receipt = - self.candidate_receipt_templates[candidate_index].clone(); - - // Make it unique. - candidate_receipt.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); - // Store the new candidate in the state - self.candidate_hashes.insert(candidate_receipt.hash(), candidate_index); - - gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); - - candidate_receipt - }) - .collect::>() - .into_iter() - .cycle(); - } - pub fn new(config: &TestConfiguration) -> Self { let config = config.clone(); + let test_authorities = config.generate_authorities(); let mut chunks = Vec::new(); let mut available_data = Vec::new(); @@ -478,6 +366,28 @@ impl TestState { candidate_receipt_templates.push(candidate_receipt); } + let block_infos: Vec = (1..=config.num_blocks) + .map(|block_num| { + let relay_block_hash = Hash::repeat_byte(block_num as u8); + new_block_import_info(relay_block_hash, block_num as BlockNumber) + }) + .collect(); + + let block_headers = (1..=config.num_blocks) + .map(|block_number| { + ( + Hash::repeat_byte(block_number as u8), + Header { + digest: Default::default(), + number: block_number as BlockNumber, + parent_hash: Default::default(), + extrinsics_root: Default::default(), + state_root: Default::default(), + }, + ) + }) + .collect::>(); + gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); let mut _self = Self { @@ -489,19 +399,139 @@ impl TestState { candidate_hashes: HashMap::new(), candidates: Vec::new().into_iter().cycle(), backed_candidates: Vec::new(), - config, - block_infos: Default::default(), + config: config.clone(), + block_infos, chunk_fetching_requests: Default::default(), signed_bitfields: Default::default(), + candidate_hashes_2: Default::default(), + block_headers, + test_authorities, }; _self.generate_candidates(); + + let mut candidate_hashes_2: HashMap> = HashMap::new(); + + // Prepare per block candidates. + // Genesis block is always finalized, so we start at 1. + for block_num in 1..=config.num_blocks { + for _ in 0..config.n_cores { + candidate_hashes_2 + .entry(Hash::repeat_byte(block_num as u8)) + .or_default() + .push(_self.next_candidate().expect("Cycle iterator")) + } + + // First candidate is our backed candidate. + _self.backed_candidates.push( + candidate_hashes_2 + .get(&Hash::repeat_byte(block_num as u8)) + .expect("just inserted above") + .first() + .expect("just inserted above") + .clone(), + ); + } + + let chunk_fetching_requests = _self + .backed_candidates() + .iter() + .map(|candidate| { + (0..config.n_validators) + .map(|index| { + ChunkFetchingRequest { + candidate_hash: candidate.hash(), + index: ValidatorIndex(index as u32), + } + .encode() + }) + .collect::>() + }) + .collect::>(); + + let signed_bitfields: HashMap> = _self + .block_infos + .iter() + .map(|block_info| { + let signing_context = + SigningContext { session_index: 0, parent_hash: block_info.hash }; + let messages = (0..config.n_validators) + .map(|index| { + let validator_public = _self + .test_authorities + .validator_public + .get(index) + .expect("All validator keys are known"); + + // Node has all the chunks in the world. + let payload: AvailabilityBitfield = + AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); + let signed_bitfield = Signed::::sign( + &_self.test_authorities.keyring.keystore(), + payload, + &signing_context, + ValidatorIndex(index as u32), + validator_public, + ) + .ok() + .flatten() + .expect("should be signed"); + + peer_bitfield_message_v2(block_info.hash, signed_bitfield) + }) + .collect::>(); + + (block_info.hash, messages) + }) + .collect(); + + _self.chunk_fetching_requests = chunk_fetching_requests; + _self.signed_bitfields = signed_bitfields; + _self.candidate_hashes_2 = candidate_hashes_2; + _self } pub fn backed_candidates(&self) -> &Vec { &self.backed_candidates } + + pub fn next_candidate(&mut self) -> Option { + let candidate = self.candidates.next(); + let candidate_hash = candidate.as_ref().unwrap().hash(); + gum::trace!(target: LOG_TARGET, "Next candidate selected {:?}", candidate_hash); + candidate + } + + /// Generate candidates to be used in the test. + fn generate_candidates(&mut self) { + let count = self.config.n_cores * self.config.num_blocks; + gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); + + // Generate all candidates + self.candidates = (0..count) + .map(|index| { + let pov_size = self.pov_sizes.next().expect("This is a cycle; qed"); + let candidate_index = *self + .pov_size_to_candidate + .get(&pov_size) + .expect("pov_size always exists; qed"); + let mut candidate_receipt = + self.candidate_receipt_templates[candidate_index].clone(); + + // Make it unique. + candidate_receipt.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); + // Store the new candidate in the state + self.candidate_hashes.insert(candidate_receipt.hash(), candidate_index); + + gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); + + candidate_receipt + }) + .collect::>() + .into_iter() + .cycle(); + } } pub async fn benchmark_availability_read( From 4ca7d3abba9a785c179921f63521713ccffe05fe Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 20 Mar 2024 17:35:32 +0100 Subject: [PATCH 20/37] Update --- .../src/lib/availability/mod.rs | 286 +----------------- .../src/lib/availability/test_state.rs | 271 +++++++++++++++++ 2 files changed, 282 insertions(+), 275 deletions(-) create mode 100644 polkadot/node/subsystem-bench/src/lib/availability/test_state.rs diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 998d82152bea..a1974bc92488 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -15,11 +15,12 @@ // along with Polkadot. If not, see . use crate::{ - configuration::{TestAuthorities, TestConfiguration}, + availability::av_store_helpers::new_av_store, + configuration::TestAuthorities, dummy_builder, environment::{TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, mock::{ - av_store::{self, MockAvailabilityStore}, + av_store::{self, MockAvailabilityStore, NetworkAvailabilityState}, chain_api::{ChainApiState, MockChainApi}, network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx}, runtime_api::{self, MockRuntimeApi}, @@ -28,12 +29,8 @@ use crate::{ network::new_network, usage::BenchmarkUsage, }; -use av_store::NetworkAvailabilityState; -use av_store_helpers::new_av_store; -use bitvec::bitvec; use colored::Colorize; use futures::{channel::oneshot, stream::FuturesUnordered, StreamExt}; -use itertools::Itertools; use parity_scale_codec::Encode; use polkadot_availability_bitfield_distribution::BitfieldDistribution; use polkadot_availability_distribution::{ @@ -44,33 +41,27 @@ use polkadot_node_core_av_store::AvailabilityStoreSubsystem; use polkadot_node_metrics::metrics::Metrics; use polkadot_node_network_protocol::{ request_response::{v1::ChunkFetchingRequest, IncomingRequest, ReqProtocolNames}, - OurView, Versioned, VersionedValidationProtocol, + OurView, }; -use polkadot_node_primitives::{AvailableData, BlockData, ErasureChunk, PoV}; use polkadot_node_subsystem::{ messages::{AllMessages, AvailabilityRecoveryMessage}, Overseer, OverseerConnector, SpawnGlue, }; -use polkadot_node_subsystem_test_helpers::{ - derive_erasure_chunks_with_proofs_and_root, mock::new_block_import_info, -}; +use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_node_subsystem_types::{ messages::{AvailabilityStoreMessage, NetworkBridgeEvent}, Span, }; -use polkadot_overseer::{metrics::Metrics as OverseerMetrics, BlockInfo, Handle as OverseerHandle}; -use polkadot_primitives::{ - AvailabilityBitfield, BlockNumber, CandidateHash, CandidateReceipt, GroupIndex, Hash, HeadData, - Header, PersistedValidationData, Signed, SigningContext, ValidatorIndex, -}; -use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; +use polkadot_overseer::{metrics::Metrics as OverseerMetrics, Handle as OverseerHandle}; +use polkadot_primitives::{GroupIndex, Hash}; use sc_network::request_responses::IncomingRequest as RawIncomingRequest; use sc_service::SpawnTaskHandle; use serde::{Deserialize, Serialize}; -use sp_core::H256; -use std::{collections::HashMap, iter::Cycle, ops::Sub, sync::Arc, time::Instant}; +use std::{ops::Sub, sync::Arc, time::Instant}; +pub use test_state::TestState; mod av_store_helpers; +mod test_state; const LOG_TARGET: &str = "subsystem-bench::availability"; @@ -293,247 +284,6 @@ pub fn prepare_availability_write_test( ) } -#[derive(Clone)] -pub struct TestState { - // Full test configuration - config: TestConfiguration, - // A cycle iterator on all PoV sizes used in the test. - pov_sizes: Cycle>, - // Generated candidate receipts to be used in the test - candidates: Cycle>, - // Map from pov size to candidate index - pov_size_to_candidate: HashMap, - // Map from generated candidate hashes to candidate index in `available_data` - // and `chunks`. - candidate_hashes: HashMap, - // Per candidate index receipts. - candidate_receipt_templates: Vec, - // Per candidate index `AvailableData` - available_data: Vec, - // Per candiadte index chunks - chunks: Vec>, - // Per relay chain block - candidate backed by our backing group - backed_candidates: Vec, - block_infos: Vec, - chunk_fetching_requests: Vec>>, - signed_bitfields: HashMap>, - block_headers: HashMap, - test_authorities: TestAuthorities, - candidate_hashes_2: HashMap>, -} - -impl TestState { - pub fn new(config: &TestConfiguration) -> Self { - let config = config.clone(); - let test_authorities = config.generate_authorities(); - - let mut chunks = Vec::new(); - let mut available_data = Vec::new(); - let mut candidate_receipt_templates = Vec::new(); - let mut pov_size_to_candidate = HashMap::new(); - - // we use it for all candidates. - let persisted_validation_data = PersistedValidationData { - parent_head: HeadData(vec![7, 8, 9]), - relay_parent_number: Default::default(), - max_pov_size: 1024, - relay_parent_storage_root: Default::default(), - }; - - // For each unique pov we create a candidate receipt. - for (index, pov_size) in config.pov_sizes().iter().cloned().unique().enumerate() { - gum::info!(target: LOG_TARGET, index, pov_size, "{}", "Generating template candidate".bright_blue()); - - let mut candidate_receipt = dummy_candidate_receipt(dummy_hash()); - let pov = PoV { block_data: BlockData(vec![index as u8; pov_size]) }; - - let new_available_data = AvailableData { - validation_data: persisted_validation_data.clone(), - pov: Arc::new(pov), - }; - - let (new_chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( - config.n_validators, - &new_available_data, - |_, _| {}, - ); - - candidate_receipt.descriptor.erasure_root = erasure_root; - - chunks.push(new_chunks); - available_data.push(new_available_data); - pov_size_to_candidate.insert(pov_size, index); - candidate_receipt_templates.push(candidate_receipt); - } - - let block_infos: Vec = (1..=config.num_blocks) - .map(|block_num| { - let relay_block_hash = Hash::repeat_byte(block_num as u8); - new_block_import_info(relay_block_hash, block_num as BlockNumber) - }) - .collect(); - - let block_headers = (1..=config.num_blocks) - .map(|block_number| { - ( - Hash::repeat_byte(block_number as u8), - Header { - digest: Default::default(), - number: block_number as BlockNumber, - parent_hash: Default::default(), - extrinsics_root: Default::default(), - state_root: Default::default(), - }, - ) - }) - .collect::>(); - - gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); - - let mut _self = Self { - available_data, - candidate_receipt_templates, - chunks, - pov_size_to_candidate, - pov_sizes: Vec::from(config.pov_sizes()).into_iter().cycle(), - candidate_hashes: HashMap::new(), - candidates: Vec::new().into_iter().cycle(), - backed_candidates: Vec::new(), - config: config.clone(), - block_infos, - chunk_fetching_requests: Default::default(), - signed_bitfields: Default::default(), - candidate_hashes_2: Default::default(), - block_headers, - test_authorities, - }; - - _self.generate_candidates(); - - let mut candidate_hashes_2: HashMap> = HashMap::new(); - - // Prepare per block candidates. - // Genesis block is always finalized, so we start at 1. - for block_num in 1..=config.num_blocks { - for _ in 0..config.n_cores { - candidate_hashes_2 - .entry(Hash::repeat_byte(block_num as u8)) - .or_default() - .push(_self.next_candidate().expect("Cycle iterator")) - } - - // First candidate is our backed candidate. - _self.backed_candidates.push( - candidate_hashes_2 - .get(&Hash::repeat_byte(block_num as u8)) - .expect("just inserted above") - .first() - .expect("just inserted above") - .clone(), - ); - } - - let chunk_fetching_requests = _self - .backed_candidates() - .iter() - .map(|candidate| { - (0..config.n_validators) - .map(|index| { - ChunkFetchingRequest { - candidate_hash: candidate.hash(), - index: ValidatorIndex(index as u32), - } - .encode() - }) - .collect::>() - }) - .collect::>(); - - let signed_bitfields: HashMap> = _self - .block_infos - .iter() - .map(|block_info| { - let signing_context = - SigningContext { session_index: 0, parent_hash: block_info.hash }; - let messages = (0..config.n_validators) - .map(|index| { - let validator_public = _self - .test_authorities - .validator_public - .get(index) - .expect("All validator keys are known"); - - // Node has all the chunks in the world. - let payload: AvailabilityBitfield = - AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); - let signed_bitfield = Signed::::sign( - &_self.test_authorities.keyring.keystore(), - payload, - &signing_context, - ValidatorIndex(index as u32), - validator_public, - ) - .ok() - .flatten() - .expect("should be signed"); - - peer_bitfield_message_v2(block_info.hash, signed_bitfield) - }) - .collect::>(); - - (block_info.hash, messages) - }) - .collect(); - - _self.chunk_fetching_requests = chunk_fetching_requests; - _self.signed_bitfields = signed_bitfields; - _self.candidate_hashes_2 = candidate_hashes_2; - - _self - } - - pub fn backed_candidates(&self) -> &Vec { - &self.backed_candidates - } - - pub fn next_candidate(&mut self) -> Option { - let candidate = self.candidates.next(); - let candidate_hash = candidate.as_ref().unwrap().hash(); - gum::trace!(target: LOG_TARGET, "Next candidate selected {:?}", candidate_hash); - candidate - } - - /// Generate candidates to be used in the test. - fn generate_candidates(&mut self) { - let count = self.config.n_cores * self.config.num_blocks; - gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", count).bright_blue()); - - // Generate all candidates - self.candidates = (0..count) - .map(|index| { - let pov_size = self.pov_sizes.next().expect("This is a cycle; qed"); - let candidate_index = *self - .pov_size_to_candidate - .get(&pov_size) - .expect("pov_size always exists; qed"); - let mut candidate_receipt = - self.candidate_receipt_templates[candidate_index].clone(); - - // Make it unique. - candidate_receipt.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); - // Store the new candidate in the state - self.candidate_hashes.insert(candidate_receipt.hash(), candidate_index); - - gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); - - candidate_receipt - }) - .collect::>() - .into_iter() - .cycle(); - } -} - pub async fn benchmark_availability_read( benchmark_name: &str, env: &mut TestEnvironment, @@ -613,7 +363,7 @@ pub async fn benchmark_availability_write( env.metrics().set_n_cores(config.n_cores); gum::info!(target: LOG_TARGET, "Seeding availability store with candidates ..."); - for backed_candidate in state.backed_candidates().clone() { + for backed_candidate in state.backed_candidates.clone() { let candidate_index = *state.candidate_hashes.get(&backed_candidate.hash()).unwrap(); let available_data = state.available_data[candidate_index].clone(); let (tx, rx) = oneshot::channel(); @@ -740,17 +490,3 @@ pub async fn benchmark_availability_write( &["availability-distribution", "bitfield-distribution", "availability-store"], ) } - -pub fn peer_bitfield_message_v2( - relay_hash: H256, - signed_bitfield: Signed, -) -> VersionedValidationProtocol { - let bitfield = polkadot_node_network_protocol::v2::BitfieldDistributionMessage::Bitfield( - relay_hash, - signed_bitfield.into(), - ); - - Versioned::V2(polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution( - bitfield, - )) -} diff --git a/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs b/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs new file mode 100644 index 000000000000..15ac7a446b15 --- /dev/null +++ b/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs @@ -0,0 +1,271 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +use crate::configuration::{TestAuthorities, TestConfiguration}; +use bitvec::bitvec; +use colored::Colorize; +use itertools::Itertools; +use parity_scale_codec::Encode; +use polkadot_node_network_protocol::{ + request_response::v1::ChunkFetchingRequest, Versioned, VersionedValidationProtocol, +}; +use polkadot_node_primitives::{AvailableData, BlockData, ErasureChunk, PoV}; +use polkadot_node_subsystem_test_helpers::{ + derive_erasure_chunks_with_proofs_and_root, mock::new_block_import_info, +}; +use polkadot_overseer::BlockInfo; +use polkadot_primitives::{ + AvailabilityBitfield, BlockNumber, CandidateHash, CandidateReceipt, Hash, HeadData, Header, + PersistedValidationData, Signed, SigningContext, ValidatorIndex, +}; +use polkadot_primitives_test_helpers::{dummy_candidate_receipt, dummy_hash}; +use sp_core::H256; +use std::{collections::HashMap, iter::Cycle, sync::Arc}; + +const LOG_TARGET: &str = "subsystem-bench::availability::test_state"; + +#[derive(Clone)] +pub struct TestState { + // Full test configuration + pub config: TestConfiguration, + // A cycle iterator on all PoV sizes used in the test. + pub pov_sizes: Cycle>, + // Generated candidate receipts to be used in the test + pub candidates: Cycle>, + // Map from pov size to candidate index + pub pov_size_to_candidate: HashMap, + // Map from generated candidate hashes to candidate index in `available_data` and `chunks`. + pub candidate_hashes: HashMap, + // Per candidate index receipts. + pub candidate_receipt_templates: Vec, + // Per candidate index `AvailableData` + pub available_data: Vec, + // Per candiadte index chunks + pub chunks: Vec>, + // Per relay chain block - candidate backed by our backing group + pub backed_candidates: Vec, + pub block_infos: Vec, + pub chunk_fetching_requests: Vec>>, + pub signed_bitfields: HashMap>, + pub block_headers: HashMap, + pub test_authorities: TestAuthorities, + pub candidate_receipts: HashMap>, +} + +impl TestState { + pub fn new(config: &TestConfiguration) -> Self { + let mut test_state = Self { + available_data: Default::default(), + candidate_receipt_templates: Default::default(), + chunks: Default::default(), + pov_size_to_candidate: Default::default(), + pov_sizes: Vec::from(config.pov_sizes()).into_iter().cycle(), + candidate_hashes: HashMap::new(), + candidates: Vec::new().into_iter().cycle(), + backed_candidates: Vec::new(), + config: config.clone(), + block_infos: Default::default(), + chunk_fetching_requests: Default::default(), + signed_bitfields: Default::default(), + candidate_receipts: Default::default(), + block_headers: Default::default(), + test_authorities: config.generate_authorities(), + }; + + // we use it for all candidates. + let persisted_validation_data = PersistedValidationData { + parent_head: HeadData(vec![7, 8, 9]), + relay_parent_number: Default::default(), + max_pov_size: 1024, + relay_parent_storage_root: Default::default(), + }; + + // For each unique pov we create a candidate receipt. + for (index, pov_size) in config.pov_sizes().iter().cloned().unique().enumerate() { + gum::info!(target: LOG_TARGET, index, pov_size, "{}", "Generating template candidate".bright_blue()); + + let mut candidate_receipt = dummy_candidate_receipt(dummy_hash()); + let pov = PoV { block_data: BlockData(vec![index as u8; pov_size]) }; + + let new_available_data = AvailableData { + validation_data: persisted_validation_data.clone(), + pov: Arc::new(pov), + }; + + let (new_chunks, erasure_root) = derive_erasure_chunks_with_proofs_and_root( + config.n_validators, + &new_available_data, + |_, _| {}, + ); + + candidate_receipt.descriptor.erasure_root = erasure_root; + + test_state.chunks.push(new_chunks); + test_state.available_data.push(new_available_data); + test_state.pov_size_to_candidate.insert(pov_size, index); + test_state.candidate_receipt_templates.push(candidate_receipt); + } + + test_state.block_infos = (1..=config.num_blocks) + .map(|block_num| { + let relay_block_hash = Hash::repeat_byte(block_num as u8); + new_block_import_info(relay_block_hash, block_num as BlockNumber) + }) + .collect(); + + test_state.block_headers = (1..=config.num_blocks) + .map(|block_number| { + ( + Hash::repeat_byte(block_number as u8), + Header { + digest: Default::default(), + number: block_number as BlockNumber, + parent_hash: Default::default(), + extrinsics_root: Default::default(), + state_root: Default::default(), + }, + ) + }) + .collect::>(); + + // Generate all candidates + let candidates_count = config.n_cores * config.num_blocks; + gum::info!(target: LOG_TARGET,"{}", format!("Pre-generating {} candidates.", candidates_count).bright_blue()); + test_state.candidates = (0..candidates_count) + .map(|index| { + let pov_size = test_state.pov_sizes.next().expect("This is a cycle; qed"); + let candidate_index = *test_state + .pov_size_to_candidate + .get(&pov_size) + .expect("pov_size always exists; qed"); + let mut candidate_receipt = + test_state.candidate_receipt_templates[candidate_index].clone(); + + // Make it unique. + candidate_receipt.descriptor.relay_parent = Hash::from_low_u64_be(index as u64); + // Store the new candidate in the state + test_state.candidate_hashes.insert(candidate_receipt.hash(), candidate_index); + + gum::debug!(target: LOG_TARGET, candidate_hash = ?candidate_receipt.hash(), "new candidate"); + + candidate_receipt + }) + .collect::>() + .into_iter() + .cycle(); + + // Prepare per block candidates. + // Genesis block is always finalized, so we start at 1. + for block_num in 1..=config.num_blocks { + for _ in 0..config.n_cores { + let receipt = test_state.next_candidate().expect("Cycle iterator"); + test_state + .candidate_receipts + .entry(Hash::repeat_byte(block_num as u8)) + .or_default() + .push(receipt); + } + + // First candidate is our backed candidate. + test_state.backed_candidates.push( + test_state + .candidate_receipts + .get(&Hash::repeat_byte(block_num as u8)) + .expect("just inserted above") + .first() + .expect("just inserted above") + .clone(), + ); + } + + test_state.chunk_fetching_requests = test_state + .backed_candidates + .iter() + .map(|candidate| { + (0..config.n_validators) + .map(|index| { + ChunkFetchingRequest { + candidate_hash: candidate.hash(), + index: ValidatorIndex(index as u32), + } + .encode() + }) + .collect::>() + }) + .collect::>(); + + test_state.signed_bitfields = test_state + .block_infos + .iter() + .map(|block_info| { + let signing_context = + SigningContext { session_index: 0, parent_hash: block_info.hash }; + let messages = (0..config.n_validators) + .map(|index| { + let validator_public = test_state + .test_authorities + .validator_public + .get(index) + .expect("All validator keys are known"); + + // Node has all the chunks in the world. + let payload: AvailabilityBitfield = + AvailabilityBitfield(bitvec![u8, bitvec::order::Lsb0; 1u8; 32]); + let signed_bitfield = Signed::::sign( + &test_state.test_authorities.keyring.keystore(), + payload, + &signing_context, + ValidatorIndex(index as u32), + validator_public, + ) + .ok() + .flatten() + .expect("should be signed"); + + peer_bitfield_message_v2(block_info.hash, signed_bitfield) + }) + .collect::>(); + + (block_info.hash, messages) + }) + .collect(); + + gum::info!(target: LOG_TARGET, "{}","Created test environment.".bright_blue()); + + test_state + } + + pub fn next_candidate(&mut self) -> Option { + let candidate = self.candidates.next(); + let candidate_hash = candidate.as_ref().unwrap().hash(); + gum::trace!(target: LOG_TARGET, "Next candidate selected {:?}", candidate_hash); + candidate + } +} + +fn peer_bitfield_message_v2( + relay_hash: H256, + signed_bitfield: Signed, +) -> VersionedValidationProtocol { + let bitfield = polkadot_node_network_protocol::v2::BitfieldDistributionMessage::Bitfield( + relay_hash, + signed_bitfield.into(), + ); + + Versioned::V2(polkadot_node_network_protocol::v2::ValidationProtocol::BitfieldDistribution( + bitfield, + )) +} From b958883a5edd9213803831bf9dda46d0720bafa8 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 20 Mar 2024 18:29:35 +0100 Subject: [PATCH 21/37] Update --- .../availability-recovery-regression-bench.rs | 39 ++--- .../src/lib/availability/mod.rs | 134 ++++++++++++++++-- 2 files changed, 145 insertions(+), 28 deletions(-) diff --git a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs index 0aafd7fe9cbf..b571376b43d5 100644 --- a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs @@ -14,9 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -//! availability-write regression tests +//! availability-read regression tests //! -//! Availability write benchmark based on Kusama parameters and scale. +//! Availability read benchmark based on Kusama parameters and scale. //! //! Subsystems involved: //! - availability-recovery @@ -27,8 +27,11 @@ use polkadot_subsystem_bench::{ TestDataAvailability, TestState, }, configuration::TestConfiguration, - utils::{warm_up_and_benchmark, WarmUpOptions}, + usage::BenchmarkUsage, }; +use std::io::Write; + +const BENCH_COUNT: usize = 50; fn main() -> Result<(), String> { let mut messages = vec![]; @@ -38,28 +41,30 @@ fn main() -> Result<(), String> { config.num_blocks = 3; config.generate_pov_sizes(); - let usage = - warm_up_and_benchmark(WarmUpOptions::new(&[("availability-recovery", 0.01)]), || { - let mut state = TestState::new(&config); - let (mut env, _protocol_config) = prepare_test( - config.clone(), - &mut state, - TestDataAvailability::Read(options.clone()), - false, - ); + let state = TestState::new(&config); + + println!("Benchmarking..."); + let usages: Vec = (0..BENCH_COUNT) + .map(|n| { + print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n)); + std::io::stdout().flush().unwrap(); + let mut env = prepare_test(&state, TestDataAvailability::Read(options.clone()), false); env.runtime().block_on(benchmark_availability_read( "data_availability_read", &mut env, - state, + state.clone(), )) - })?; - println!("{}", usage); + }) + .collect(); + println!("\rDone!{}", " ".repeat(BENCH_COUNT)); + let average_usage = BenchmarkUsage::average(&usages); + println!("{}", average_usage); - messages.extend(usage.check_network_usage(&[ + messages.extend(average_usage.check_network_usage(&[ ("Received from peers", 307200.000, 0.05), ("Sent to peers", 1.667, 0.05), ])); - messages.extend(usage.check_cpu_usage(&[("availability-recovery", 11.500, 0.05)])); + messages.extend(average_usage.check_cpu_usage(&[("availability-recovery", 11.500, 0.05)])); if messages.is_empty() { Ok(()) diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index a1974bc92488..0fb6d1f5ac95 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -16,7 +16,6 @@ use crate::{ availability::av_store_helpers::new_av_store, - configuration::TestAuthorities, dummy_builder, environment::{TestEnvironment, TestEnvironmentDependencies, GENESIS_HASH}, mock::{ @@ -137,18 +136,131 @@ fn build_overseer_for_availability_write( (overseer, OverseerHandle::new(raw_handle)) } +pub fn prepare_test( + state: &TestState, + mode: TestDataAvailability, + with_prometheus_endpoint: bool, +) -> TestEnvironment { + let dependencies = TestEnvironmentDependencies::default(); + + let (collation_req_receiver, _collation_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + + let (pov_req_receiver, _pov_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + + let (chunk_req_receiver, chunk_req_cfg) = + IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + + let availability_state = NetworkAvailabilityState { + candidate_hashes: state.candidate_hashes.clone(), + available_data: state.available_data.clone(), + chunks: state.chunks.clone(), + }; + let (network, network_interface, network_receiver) = new_network( + &state.config, + &dependencies, + &state.test_authorities, + vec![Arc::new(availability_state.clone())], + ); + + let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( + network.clone(), + network_interface.subsystem_sender(), + state.test_authorities.clone(), + ); + + let network_bridge_rx = + network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); + + let runtime_api = runtime_api::MockRuntimeApi::new( + state.config.clone(), + state.test_authorities.clone(), + state.candidate_receipts.clone(), + Default::default(), + Default::default(), + 0, + ); + + let (overseer, overseer_handle) = match &mode { + TestDataAvailability::Read(options) => { + let use_fast_path = options.fetch_from_backers; + + let subsystem = if use_fast_path { + AvailabilityRecoverySubsystem::with_fast_path( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + } else { + AvailabilityRecoverySubsystem::with_chunks_only( + collation_req_receiver, + Metrics::try_register(&dependencies.registry).unwrap(), + ) + }; + + // Use a mocked av-store. + let av_store = av_store::MockAvailabilityStore::new( + state.chunks.clone(), + state.candidate_hashes.clone(), + ); + + build_overseer_for_availability_read( + dependencies.task_manager.spawn_handle(), + runtime_api.clone(), + av_store, + (network_bridge_tx, network_bridge_rx), + subsystem, + &dependencies, + ) + }, + TestDataAvailability::Write => { + let availability_distribution = AvailabilityDistributionSubsystem::new( + state.test_authorities.keyring.keystore(), + IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, + Metrics::try_register(&dependencies.registry).unwrap(), + ); + + let chain_api_state = ChainApiState { block_headers: state.block_headers.clone() }; + let chain_api = MockChainApi::new(chain_api_state); + let bitfield_distribution = + BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); + build_overseer_for_availability_write( + dependencies.task_manager.spawn_handle(), + runtime_api.clone(), + (network_bridge_tx, network_bridge_rx), + availability_distribution, + chain_api, + new_av_store(&dependencies), + bitfield_distribution, + &dependencies, + ) + }, + }; + + TestEnvironment::new( + dependencies, + state.config.clone(), + network, + overseer, + overseer_handle, + state.test_authorities.clone(), + with_prometheus_endpoint, + ) +} + /// Takes a test configuration and uses it to create the `TestEnvironment`. pub fn prepare_availability_read_test( state: &TestState, - test_authorities: &TestAuthorities, options: &DataAvailabilityReadOptions, with_prometheus_endpoint: bool, ) -> TestEnvironment { - let (collation_req_receiver, _) = + let (collation_req_receiver, _collation_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - let (_, chunk_req_cfg) = IncomingRequest::::get_config_receiver( - &ReqProtocolNames::new(GENESIS_HASH, None), - ); + let (_chunk_req_receiver, chunk_req_cfg) = + IncomingRequest::::get_config_receiver(&ReqProtocolNames::new( + GENESIS_HASH, + None, + )); let dependencies = TestEnvironmentDependencies::default(); let availability_state = NetworkAvailabilityState { @@ -159,13 +271,13 @@ pub fn prepare_availability_read_test( let (network, network_interface, network_receiver) = new_network( &state.config, &dependencies, - &test_authorities, + &state.test_authorities, vec![Arc::new(availability_state.clone())], ); let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( network.clone(), network_interface.subsystem_sender(), - test_authorities.clone(), + state.test_authorities.clone(), ); let network_bridge_rx = network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); @@ -188,7 +300,7 @@ pub fn prepare_availability_read_test( let runtime_api = runtime_api::MockRuntimeApi::new( state.config.clone(), state.test_authorities.clone(), - state.candidate_hashes_2.clone(), + state.candidate_receipts.clone(), Default::default(), Default::default(), 0, @@ -209,7 +321,7 @@ pub fn prepare_availability_read_test( network, overseer, overseer_handle, - test_authorities.clone(), + state.test_authorities.clone(), with_prometheus_endpoint, ) } @@ -257,7 +369,7 @@ pub fn prepare_availability_write_test( let runtime_api = runtime_api::MockRuntimeApi::new( state.config.clone(), state.test_authorities.clone(), - state.candidate_hashes_2.clone(), + state.candidate_receipts.clone(), Default::default(), Default::default(), 0, From afea297036fb3ceb08420b37c8a1efd0d3aa4512 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 12:14:17 +0100 Subject: [PATCH 22/37] Update --- ...ilability-distribution-regression-bench.rs | 8 +- .../availability-recovery-regression-bench.rs | 2 +- .../src/lib/availability/mod.rs | 157 +----------------- 3 files changed, 12 insertions(+), 155 deletions(-) diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs index f209b9c27acd..12d4a196f8cc 100644 --- a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs @@ -24,7 +24,7 @@ //! - availability-store use polkadot_subsystem_bench::{ - availability::{benchmark_availability_write, prepare_availability_write_test, TestState}, + availability::{benchmark_availability_write, prepare_test, TestState}, configuration::TestConfiguration, usage::BenchmarkUsage, }; @@ -47,7 +47,11 @@ fn main() -> Result<(), String> { .map(|n| { print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n)); std::io::stdout().flush().unwrap(); - let mut env = prepare_availability_write_test(&state, false); + let mut env = prepare_test( + &state, + polkadot_subsystem_bench::availability::TestDataAvailability::Write, + false, + ); env.runtime().block_on(benchmark_availability_write( "data_availability_write", &mut env, diff --git a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs index b571376b43d5..2beef9defb14 100644 --- a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs @@ -52,7 +52,7 @@ fn main() -> Result<(), String> { env.runtime().block_on(benchmark_availability_read( "data_availability_read", &mut env, - state.clone(), + &state, )) }) .collect(); diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 0fb6d1f5ac95..a11e1c79a1ff 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -39,7 +39,7 @@ use polkadot_availability_recovery::AvailabilityRecoverySubsystem; use polkadot_node_core_av_store::AvailabilityStoreSubsystem; use polkadot_node_metrics::metrics::Metrics; use polkadot_node_network_protocol::{ - request_response::{v1::ChunkFetchingRequest, IncomingRequest, ReqProtocolNames}, + request_response::{IncomingRequest, ReqProtocolNames}, OurView, }; use polkadot_node_subsystem::{ @@ -169,7 +169,6 @@ pub fn prepare_test( network_interface.subsystem_sender(), state.test_authorities.clone(), ); - let network_bridge_rx = network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); @@ -248,158 +247,10 @@ pub fn prepare_test( ) } -/// Takes a test configuration and uses it to create the `TestEnvironment`. -pub fn prepare_availability_read_test( - state: &TestState, - options: &DataAvailabilityReadOptions, - with_prometheus_endpoint: bool, -) -> TestEnvironment { - let (collation_req_receiver, _collation_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - let (_chunk_req_receiver, chunk_req_cfg) = - IncomingRequest::::get_config_receiver(&ReqProtocolNames::new( - GENESIS_HASH, - None, - )); - - let dependencies = TestEnvironmentDependencies::default(); - let availability_state = NetworkAvailabilityState { - candidate_hashes: state.candidate_hashes.clone(), - available_data: state.available_data.clone(), - chunks: state.chunks.clone(), - }; - let (network, network_interface, network_receiver) = new_network( - &state.config, - &dependencies, - &state.test_authorities, - vec![Arc::new(availability_state.clone())], - ); - let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( - network.clone(), - network_interface.subsystem_sender(), - state.test_authorities.clone(), - ); - let network_bridge_rx = - network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); - - let subsystem = if options.fetch_from_backers { - AvailabilityRecoverySubsystem::with_fast_path( - collation_req_receiver, - Metrics::try_register(&dependencies.registry).unwrap(), - ) - } else { - AvailabilityRecoverySubsystem::with_chunks_only( - collation_req_receiver, - Metrics::try_register(&dependencies.registry).unwrap(), - ) - }; - - // Use a mocked av-store. - let av_store = - av_store::MockAvailabilityStore::new(state.chunks.clone(), state.candidate_hashes.clone()); - let runtime_api = runtime_api::MockRuntimeApi::new( - state.config.clone(), - state.test_authorities.clone(), - state.candidate_receipts.clone(), - Default::default(), - Default::default(), - 0, - ); - - let (overseer, overseer_handle) = build_overseer_for_availability_read( - dependencies.task_manager.spawn_handle(), - runtime_api, - av_store, - (network_bridge_tx, network_bridge_rx), - subsystem, - &dependencies, - ); - - TestEnvironment::new( - dependencies, - state.config.clone(), - network, - overseer, - overseer_handle, - state.test_authorities.clone(), - with_prometheus_endpoint, - ) -} - -/// Takes a test configuration and uses it to create the `TestEnvironment`. -pub fn prepare_availability_write_test( - state: &TestState, - with_prometheus_endpoint: bool, -) -> TestEnvironment { - let (pov_req_receiver, _) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - let (chunk_req_receiver, chunk_req_cfg) = - IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - - let dependencies = TestEnvironmentDependencies::default(); - let availability_state = NetworkAvailabilityState { - candidate_hashes: state.candidate_hashes.clone(), - available_data: state.available_data.clone(), - chunks: state.chunks.clone(), - }; - let (network, network_interface, network_receiver) = new_network( - &state.config, - &dependencies, - &state.test_authorities, - vec![Arc::new(availability_state.clone())], - ); - let network_bridge_tx = network_bridge::MockNetworkBridgeTx::new( - network.clone(), - network_interface.subsystem_sender(), - state.test_authorities.clone(), - ); - let network_bridge_rx = - network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); - - let availability_distribution = AvailabilityDistributionSubsystem::new( - state.test_authorities.keyring.keystore(), - IncomingRequestReceivers { pov_req_receiver, chunk_req_receiver }, - Metrics::try_register(&dependencies.registry).unwrap(), - ); - - let chain_api_state = ChainApiState { block_headers: state.block_headers.clone() }; - let chain_api = MockChainApi::new(chain_api_state); - let bitfield_distribution = - BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); - let runtime_api = runtime_api::MockRuntimeApi::new( - state.config.clone(), - state.test_authorities.clone(), - state.candidate_receipts.clone(), - Default::default(), - Default::default(), - 0, - ); - let (overseer, overseer_handle) = build_overseer_for_availability_write( - dependencies.task_manager.spawn_handle(), - runtime_api, - (network_bridge_tx, network_bridge_rx), - availability_distribution, - chain_api, - new_av_store(&dependencies), - bitfield_distribution, - &dependencies, - ); - - TestEnvironment::new( - dependencies, - state.config.clone(), - network, - overseer, - overseer_handle, - state.test_authorities.clone(), - with_prometheus_endpoint, - ) -} - pub async fn benchmark_availability_read( benchmark_name: &str, env: &mut TestEnvironment, - mut state: TestState, + state: &TestState, ) -> BenchmarkUsage { let config = env.config().clone(); @@ -412,6 +263,8 @@ pub async fn benchmark_availability_read( env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); + let mut candidates = state.candidates.clone(); + for block_num in 1..=env.config().num_blocks { gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num, env.config().num_blocks); env.metrics().set_current_block(block_num); @@ -419,7 +272,7 @@ pub async fn benchmark_availability_read( let block_start_ts = Instant::now(); for candidate_num in 0..config.n_cores as u64 { let candidate = - state.next_candidate().expect("We always send up to n_cores*num_blocks; qed"); + candidates.next().expect("We always send up to n_cores*num_blocks; qed"); let (tx, rx) = oneshot::channel(); batch.push(rx); From 8cf35fe7f8dbd23a8c52ac4a91d7766bee517d24 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 12:22:50 +0100 Subject: [PATCH 23/37] Move to benches --- .gitlab/pipeline/test.yml | 2 +- polkadot/node/network/availability-distribution/Cargo.toml | 4 ++-- .../availability-distribution-regression-bench.rs | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename polkadot/node/network/availability-distribution/{tests => benches}/availability-distribution-regression-bench.rs (100%) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 2cf8403cf8f8..476ac6333f58 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -503,7 +503,7 @@ subsystem-regression-tests: - .run-immediately script: - cargo bench --profile=testnet -p polkadot-availability-recovery --bench availability-recovery-regression-bench --features subsystem-benchmarks - - cargo test --profile=testnet -p polkadot-availability-distribution --test availability-distribution-regression-bench --features subsystem-benchmarks + - cargo bench --profile=testnet -p polkadot-availability-distribution --bench availability-distribution-regression-bench --features subsystem-benchmarks tags: - benchmark allow_failure: true diff --git a/polkadot/node/network/availability-distribution/Cargo.toml b/polkadot/node/network/availability-distribution/Cargo.toml index 182d92cb1631..ac606bd377f7 100644 --- a/polkadot/node/network/availability-distribution/Cargo.toml +++ b/polkadot/node/network/availability-distribution/Cargo.toml @@ -39,9 +39,9 @@ polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" } polkadot-subsystem-bench = { path = "../../subsystem-bench" } -[[test]] +[[bench]] name = "availability-distribution-regression-bench" -path = "tests/availability-distribution-regression-bench.rs" +path = "benches/availability-distribution-regression-bench.rs" harness = false required-features = ["subsystem-benchmarks"] diff --git a/polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs similarity index 100% rename from polkadot/node/network/availability-distribution/tests/availability-distribution-regression-bench.rs rename to polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs From 8c9723e36580430033b644af4218f6d7f5e74193 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 12:58:10 +0100 Subject: [PATCH 24/37] Fix clippy errors --- polkadot/node/subsystem-bench/src/lib/availability/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index a11e1c79a1ff..1e808eeef654 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -357,7 +357,7 @@ pub async fn benchmark_availability_write( env.metrics().set_current_block(block_num); let block_start_ts = Instant::now(); - let relay_block_hash = block_info.hash.clone(); + let relay_block_hash = block_info.hash; env.import_block(block_info.clone()).await; // Inform bitfield distribution about our view of current test block @@ -392,7 +392,7 @@ pub async fn benchmark_availability_write( None } }) - .filter_map(|v| v); + .flatten(); gum::info!(target: LOG_TARGET, "Waiting for all emulated peers to receive their chunk from us ..."); From 2131382db6d7dd803be0b04f75d255653b91aa00 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 15:25:48 +0100 Subject: [PATCH 25/37] Update --- .../src/lib/availability/mod.rs | 18 ++++++++---------- .../src/lib/availability/test_state.rs | 9 +-------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 1e808eeef654..245257c88a11 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -46,13 +46,12 @@ use polkadot_node_subsystem::{ messages::{AllMessages, AvailabilityRecoveryMessage}, Overseer, OverseerConnector, SpawnGlue, }; -use polkadot_node_subsystem_test_helpers::mock::new_block_import_info; use polkadot_node_subsystem_types::{ messages::{AvailabilityStoreMessage, NetworkBridgeEvent}, Span, }; use polkadot_overseer::{metrics::Metrics as OverseerMetrics, Handle as OverseerHandle}; -use polkadot_primitives::{GroupIndex, Hash}; +use polkadot_primitives::GroupIndex; use sc_network::request_responses::IncomingRequest as RawIncomingRequest; use sc_service::SpawnTaskHandle; use serde::{Deserialize, Serialize}; @@ -254,22 +253,21 @@ pub async fn benchmark_availability_read( ) -> BenchmarkUsage { let config = env.config().clone(); - env.import_block(new_block_import_info(Hash::repeat_byte(1), 1)).await; - - let test_start = Instant::now(); - let mut batch = FuturesUnordered::new(); - let mut availability_bytes = 0u128; - env.metrics().set_n_validators(config.n_validators); env.metrics().set_n_cores(config.n_cores); + let mut batch = FuturesUnordered::new(); + let mut availability_bytes = 0u128; let mut candidates = state.candidates.clone(); - - for block_num in 1..=env.config().num_blocks { + let test_start = Instant::now(); + for block_info in state.block_infos.iter() { + let block_num = block_info.number as usize; gum::info!(target: LOG_TARGET, "Current block {}/{}", block_num, env.config().num_blocks); env.metrics().set_current_block(block_num); let block_start_ts = Instant::now(); + env.import_block(block_info.clone()).await; + for candidate_num in 0..config.n_cores as u64 { let candidate = candidates.next().expect("We always send up to n_cores*num_blocks; qed"); diff --git a/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs b/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs index 15ac7a446b15..7494bd4fe9f3 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs @@ -171,7 +171,7 @@ impl TestState { // Genesis block is always finalized, so we start at 1. for block_num in 1..=config.num_blocks { for _ in 0..config.n_cores { - let receipt = test_state.next_candidate().expect("Cycle iterator"); + let receipt = test_state.candidates.next().expect("Cycle iterator"); test_state .candidate_receipts .entry(Hash::repeat_byte(block_num as u8)) @@ -247,13 +247,6 @@ impl TestState { test_state } - - pub fn next_candidate(&mut self) -> Option { - let candidate = self.candidates.next(); - let candidate_hash = candidate.as_ref().unwrap().hash(); - gum::trace!(target: LOG_TARGET, "Next candidate selected {:?}", candidate_hash); - candidate - } } fn peer_bitfield_message_v2( From f23ff460c23b289b4b06a3fbbd1fbb75f2efd58b Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 15:51:26 +0100 Subject: [PATCH 26/37] Update --- ...ilability-distribution-regression-bench.rs | 2 +- .../availability-recovery-regression-bench.rs | 3 +- .../src/lib/availability/mod.rs | 35 ++++++++++--------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs index 12d4a196f8cc..b944efd9501b 100644 --- a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs @@ -47,7 +47,7 @@ fn main() -> Result<(), String> { .map(|n| { print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n)); std::io::stdout().flush().unwrap(); - let mut env = prepare_test( + let (mut env, _cfgs) = prepare_test( &state, polkadot_subsystem_bench::availability::TestDataAvailability::Write, false, diff --git a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs index 2beef9defb14..e34fe3a0353f 100644 --- a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs @@ -48,7 +48,8 @@ fn main() -> Result<(), String> { .map(|n| { print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n)); std::io::stdout().flush().unwrap(); - let mut env = prepare_test(&state, TestDataAvailability::Read(options.clone()), false); + let (mut env, _cfgs) = + prepare_test(&state, TestDataAvailability::Read(options.clone()), false); env.runtime().block_on(benchmark_availability_read( "data_availability_read", &mut env, diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 245257c88a11..38b1c97cce61 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -52,7 +52,7 @@ use polkadot_node_subsystem_types::{ }; use polkadot_overseer::{metrics::Metrics as OverseerMetrics, Handle as OverseerHandle}; use polkadot_primitives::GroupIndex; -use sc_network::request_responses::IncomingRequest as RawIncomingRequest; +use sc_network::request_responses::{IncomingRequest as RawIncomingRequest, ProtocolConfig}; use sc_service::SpawnTaskHandle; use serde::{Deserialize, Serialize}; use std::{ops::Sub, sync::Arc, time::Instant}; @@ -139,18 +139,16 @@ pub fn prepare_test( state: &TestState, mode: TestDataAvailability, with_prometheus_endpoint: bool, -) -> TestEnvironment { - let dependencies = TestEnvironmentDependencies::default(); - - let (collation_req_receiver, _collation_req_cfg) = +) -> (TestEnvironment, Vec) { + let (collation_req_receiver, collation_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - - let (pov_req_receiver, _pov_req_cfg) = + let (pov_req_receiver, pov_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); - let (chunk_req_receiver, chunk_req_cfg) = IncomingRequest::get_config_receiver(&ReqProtocolNames::new(GENESIS_HASH, None)); + let req_cfgs = vec![collation_req_cfg, pov_req_cfg]; + let dependencies = TestEnvironmentDependencies::default(); let availability_state = NetworkAvailabilityState { candidate_hashes: state.candidate_hashes.clone(), available_data: state.available_data.clone(), @@ -169,7 +167,7 @@ pub fn prepare_test( state.test_authorities.clone(), ); let network_bridge_rx = - network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg.clone())); + network_bridge::MockNetworkBridgeRx::new(network_receiver, Some(chunk_req_cfg)); let runtime_api = runtime_api::MockRuntimeApi::new( state.config.clone(), @@ -235,14 +233,17 @@ pub fn prepare_test( }, }; - TestEnvironment::new( - dependencies, - state.config.clone(), - network, - overseer, - overseer_handle, - state.test_authorities.clone(), - with_prometheus_endpoint, + ( + TestEnvironment::new( + dependencies, + state.config.clone(), + network, + overseer, + overseer_handle, + state.test_authorities.clone(), + with_prometheus_endpoint, + ), + req_cfgs, ) } From 0c9c44a31654dc2e3a849e8bdbdbd8fd01cf7452 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 16:21:04 +0100 Subject: [PATCH 27/37] Fix cli --- polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index deb351360d74..b2c68ca3490e 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -138,7 +138,6 @@ impl BenchCli { TestObjective::DataAvailabilityRead(opts) => { let mut state = availability::TestState::new(&test_config); let (mut env, _protocol_config) = availability::prepare_test( - test_config, &mut state, availability::TestDataAvailability::Read(opts), true, @@ -146,13 +145,12 @@ impl BenchCli { env.runtime().block_on(availability::benchmark_availability_read( &benchmark_name, &mut env, - state, + &state, )) }, TestObjective::DataAvailabilityWrite => { let mut state = availability::TestState::new(&test_config); let (mut env, _protocol_config) = availability::prepare_test( - test_config, &mut state, availability::TestDataAvailability::Write, true, @@ -160,7 +158,7 @@ impl BenchCli { env.runtime().block_on(availability::benchmark_availability_write( &benchmark_name, &mut env, - state, + &state, )) }, TestObjective::ApprovalVoting(ref options) => { From 7837b43298f8ed2fb481e9dc83321dcce47caff9 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 16:45:00 +0100 Subject: [PATCH 28/37] Fix clippy --- .../src/lib/availability/mod.rs | 43 +++++++++---------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 38b1c97cce61..4cb53c994246 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -369,29 +369,26 @@ pub async fn benchmark_availability_write( // Request chunks of our own backed candidate from all other validators. let payloads = state.chunk_fetching_requests.get(block_num - 1).expect("pregenerated"); - let receivers = (1..config.n_validators) - .map(|index| { - let (pending_response, pending_response_receiver) = oneshot::channel(); - - let peer_id = - *env.authorities().peer_ids.get(index).expect("all validators have ids"); - let payload = payloads.get(index).expect("pregenerated").clone(); - let request = RawIncomingRequest { peer: peer_id, payload, pending_response }; - let peer = env - .authorities() - .validator_authority_id - .get(index) - .expect("all validators have keys"); - - if env.network().is_peer_connected(peer) && - env.network().send_request_from_peer(peer, request).is_ok() - { - Some(pending_response_receiver) - } else { - None - } - }) - .flatten(); + let receivers = (1..config.n_validators).filter_map(|index| { + let (pending_response, pending_response_receiver) = oneshot::channel(); + + let peer_id = *env.authorities().peer_ids.get(index).expect("all validators have ids"); + let payload = payloads.get(index).expect("pregenerated").clone(); + let request = RawIncomingRequest { peer: peer_id, payload, pending_response }; + let peer = env + .authorities() + .validator_authority_id + .get(index) + .expect("all validators have keys"); + + if env.network().is_peer_connected(peer) && + env.network().send_request_from_peer(peer, request).is_ok() + { + Some(pending_response_receiver) + } else { + None + } + }); gum::info!(target: LOG_TARGET, "Waiting for all emulated peers to receive their chunk from us ..."); From 6fba29362fd861cbf3c93b830b887e02511235f6 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 16:54:54 +0100 Subject: [PATCH 29/37] Fix clippy --- polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index b2c68ca3490e..82df4390f0a1 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -138,7 +138,7 @@ impl BenchCli { TestObjective::DataAvailabilityRead(opts) => { let mut state = availability::TestState::new(&test_config); let (mut env, _protocol_config) = availability::prepare_test( - &mut state, + &state, availability::TestDataAvailability::Read(opts), true, ); @@ -151,7 +151,7 @@ impl BenchCli { TestObjective::DataAvailabilityWrite => { let mut state = availability::TestState::new(&test_config); let (mut env, _protocol_config) = availability::prepare_test( - &mut state, + &state, availability::TestDataAvailability::Write, true, ); From 50562562fa41bc360f29675868417076144583db Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 17:02:02 +0100 Subject: [PATCH 30/37] Fix clippy --- polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs index 82df4390f0a1..10953b6c7839 100644 --- a/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs +++ b/polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs @@ -136,7 +136,7 @@ impl BenchCli { let usage = match objective { TestObjective::DataAvailabilityRead(opts) => { - let mut state = availability::TestState::new(&test_config); + let state = availability::TestState::new(&test_config); let (mut env, _protocol_config) = availability::prepare_test( &state, availability::TestDataAvailability::Read(opts), @@ -149,7 +149,7 @@ impl BenchCli { )) }, TestObjective::DataAvailabilityWrite => { - let mut state = availability::TestState::new(&test_config); + let state = availability::TestState::new(&test_config); let (mut env, _protocol_config) = availability::prepare_test( &state, availability::TestDataAvailability::Write, From 30caa9601b8e647dec3aeb11c3de19bfa65f00b6 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 21 Mar 2024 17:48:50 +0100 Subject: [PATCH 31/37] Update values --- .../availability-distribution-regression-bench.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs index b944efd9501b..2a2bf3b53fde 100644 --- a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs @@ -64,14 +64,14 @@ fn main() -> Result<(), String> { println!("{}", average_usage); messages.extend(average_usage.check_network_usage(&[ - ("Received from peers", 433.3, 0.05), - ("Sent to peers", 18480.0, 0.05), + ("Received from peers", 433.3, 0.01), + ("Sent to peers", 18480.0, 0.01), ])); messages.extend(average_usage.check_cpu_usage(&[ - ("availability-distribution", 0.011, 0.05), - ("bitfield-distribution", 0.050, 0.05), - ("availability-store", 0.159, 0.05), + ("availability-distribution", 0.012, 0.05), + ("availability-store", 0.153, 0.05), + ("bitfield-distribution", 0.026, 0.05), ])); if messages.is_empty() { From 6ad77004d96eb4c2e92beea72f602bdfc7922df1 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 22 Mar 2024 12:54:48 +0100 Subject: [PATCH 32/37] Update test state --- .../src/lib/availability/test_state.rs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs b/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs index 7494bd4fe9f3..c328ffedf916 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/test_state.rs @@ -57,11 +57,17 @@ pub struct TestState { pub chunks: Vec>, // Per relay chain block - candidate backed by our backing group pub backed_candidates: Vec, + // Relay chain block infos pub block_infos: Vec, + // Chung fetching requests for backed candidates pub chunk_fetching_requests: Vec>>, + // Pregenerated signed availability bitfields pub signed_bitfields: HashMap>, + // Relay chain block headers pub block_headers: HashMap, + // Authority keys for the network emulation. pub test_authorities: TestAuthorities, + // Map from generated candidate receipts pub candidate_receipts: HashMap>, } @@ -126,14 +132,16 @@ impl TestState { }) .collect(); - test_state.block_headers = (1..=config.num_blocks) - .map(|block_number| { + test_state.block_headers = test_state + .block_infos + .iter() + .map(|info| { ( - Hash::repeat_byte(block_number as u8), + info.hash, Header { digest: Default::default(), - number: block_number as BlockNumber, - parent_hash: Default::default(), + number: info.number, + parent_hash: info.parent_hash, extrinsics_root: Default::default(), state_root: Default::default(), }, @@ -169,21 +177,17 @@ impl TestState { // Prepare per block candidates. // Genesis block is always finalized, so we start at 1. - for block_num in 1..=config.num_blocks { + for info in test_state.block_infos.iter() { for _ in 0..config.n_cores { let receipt = test_state.candidates.next().expect("Cycle iterator"); - test_state - .candidate_receipts - .entry(Hash::repeat_byte(block_num as u8)) - .or_default() - .push(receipt); + test_state.candidate_receipts.entry(info.hash).or_default().push(receipt); } // First candidate is our backed candidate. test_state.backed_candidates.push( test_state .candidate_receipts - .get(&Hash::repeat_byte(block_num as u8)) + .get(&info.hash) .expect("just inserted above") .first() .expect("just inserted above") From 8c2a645992ccea2338fbe1c37fad2e75e9d47020 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 22 Mar 2024 12:57:24 +0100 Subject: [PATCH 33/37] Update values --- .../benches/availability-distribution-regression-bench.rs | 1 + .../benches/availability-recovery-regression-bench.rs | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs index 2a2bf3b53fde..89485cfda002 100644 --- a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs @@ -63,6 +63,7 @@ fn main() -> Result<(), String> { let average_usage = BenchmarkUsage::average(&usages); println!("{}", average_usage); + // We don't expect any other values for received and sent messages.extend(average_usage.check_network_usage(&[ ("Received from peers", 433.3, 0.01), ("Sent to peers", 18480.0, 0.01), diff --git a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs index e34fe3a0353f..1a89ca3a3937 100644 --- a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs @@ -61,9 +61,10 @@ fn main() -> Result<(), String> { let average_usage = BenchmarkUsage::average(&usages); println!("{}", average_usage); + // We don't expect any other values for received and sent messages.extend(average_usage.check_network_usage(&[ - ("Received from peers", 307200.000, 0.05), - ("Sent to peers", 1.667, 0.05), + ("Received from peers", 307200.000, 0.01), + ("Sent to peers", 1.667, 0.01), ])); messages.extend(average_usage.check_cpu_usage(&[("availability-recovery", 11.500, 0.05)])); From a17a8cdbaaefc5d4fe78cee1ddffc8306f37f265 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 22 Mar 2024 14:46:47 +0100 Subject: [PATCH 34/37] Remove redundant clone --- polkadot/node/subsystem-bench/src/lib/availability/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs index 4cb53c994246..765afdd5912b 100644 --- a/polkadot/node/subsystem-bench/src/lib/availability/mod.rs +++ b/polkadot/node/subsystem-bench/src/lib/availability/mod.rs @@ -202,7 +202,7 @@ pub fn prepare_test( build_overseer_for_availability_read( dependencies.task_manager.spawn_handle(), - runtime_api.clone(), + runtime_api, av_store, (network_bridge_tx, network_bridge_rx), subsystem, @@ -222,7 +222,7 @@ pub fn prepare_test( BitfieldDistribution::new(Metrics::try_register(&dependencies.registry).unwrap()); build_overseer_for_availability_write( dependencies.task_manager.spawn_handle(), - runtime_api.clone(), + runtime_api, (network_bridge_tx, network_bridge_rx), availability_distribution, chain_api, From 685fc3360c9127105d64c67332e87f1e02337a55 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 22 Mar 2024 17:52:44 +0100 Subject: [PATCH 35/37] Address PR review --- .../availability-distribution-regression-bench.rs | 8 ++++---- .../benches/availability-recovery-regression-bench.rs | 9 +++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs index 89485cfda002..2b1af40e4468 100644 --- a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs @@ -63,12 +63,12 @@ fn main() -> Result<(), String> { let average_usage = BenchmarkUsage::average(&usages); println!("{}", average_usage); - // We don't expect any other values for received and sent + // We expect no variance for received and sent + // but use 0.001 because we operate with floats messages.extend(average_usage.check_network_usage(&[ - ("Received from peers", 433.3, 0.01), - ("Sent to peers", 18480.0, 0.01), + ("Received from peers", 307200.000, 0.001), + ("Sent to peers", 1.667, 0.001), ])); - messages.extend(average_usage.check_cpu_usage(&[ ("availability-distribution", 0.012, 0.05), ("availability-store", 0.153, 0.05), diff --git a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs index 1a89ca3a3937..0d86b1d4e78c 100644 --- a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs @@ -31,7 +31,7 @@ use polkadot_subsystem_bench::{ }; use std::io::Write; -const BENCH_COUNT: usize = 50; +const BENCH_COUNT: usize = 1; fn main() -> Result<(), String> { let mut messages = vec![]; @@ -61,10 +61,11 @@ fn main() -> Result<(), String> { let average_usage = BenchmarkUsage::average(&usages); println!("{}", average_usage); - // We don't expect any other values for received and sent + // We expect no variance for received and sent + // but use 0.001 because we operate with floats messages.extend(average_usage.check_network_usage(&[ - ("Received from peers", 307200.000, 0.01), - ("Sent to peers", 1.667, 0.01), + ("Received from peers", 307200.000, 0.001), + ("Sent to peers", 1.667, 0.001), ])); messages.extend(average_usage.check_cpu_usage(&[("availability-recovery", 11.500, 0.05)])); From 514f6cb37211ec0ea61b9355341e99f68162d70c Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 22 Mar 2024 18:06:49 +0100 Subject: [PATCH 36/37] Revert accidental change --- .../benches/availability-recovery-regression-bench.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs index 0d86b1d4e78c..5e8b81be82dd 100644 --- a/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs +++ b/polkadot/node/network/availability-recovery/benches/availability-recovery-regression-bench.rs @@ -31,7 +31,7 @@ use polkadot_subsystem_bench::{ }; use std::io::Write; -const BENCH_COUNT: usize = 1; +const BENCH_COUNT: usize = 50; fn main() -> Result<(), String> { let mut messages = vec![]; From 4dae323e6fc896944853a9be55badccb2314c356 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 25 Mar 2024 09:47:27 +0100 Subject: [PATCH 37/37] Revert accidental change --- .../benches/availability-distribution-regression-bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs index 2b1af40e4468..019eb1222082 100644 --- a/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs +++ b/polkadot/node/network/availability-distribution/benches/availability-distribution-regression-bench.rs @@ -66,8 +66,8 @@ fn main() -> Result<(), String> { // We expect no variance for received and sent // but use 0.001 because we operate with floats messages.extend(average_usage.check_network_usage(&[ - ("Received from peers", 307200.000, 0.001), - ("Sent to peers", 1.667, 0.001), + ("Received from peers", 433.3, 0.001), + ("Sent to peers", 18480.0, 0.001), ])); messages.extend(average_usage.check_cpu_usage(&[ ("availability-distribution", 0.012, 0.05),