From 1d0ff37586b350fd8bbaafb47574322fd4e4e62d Mon Sep 17 00:00:00 2001
From: Marcin S <marcin@realemail.net>
Date: Fri, 29 Sep 2023 12:42:19 +0200
Subject: [PATCH 01/11] PVF: Add worker check during tests and benches

Also:

- Use an env var for the node version to avoid unnecessary dependencies on
  polkadot-cli
- Add a benchmark for preparation through the host
---
 .cargo/config.toml                            |  11 ++
 Cargo.lock                                    |   4 +-
 polkadot/cli/src/cli.rs                       |  10 --
 polkadot/cli/src/command.rs                   |  11 +-
 polkadot/node/core/pvf/Cargo.toml             |  10 +-
 .../benches/host_prepare_kusama_runtime.rs    | 129 ++++++++++++++++++
 .../node/core/pvf/common/src/worker/mod.rs    |  13 +-
 polkadot/node/core/pvf/src/lib.rs             |  13 ++
 polkadot/node/core/pvf/src/testing.rs         |  58 +++++++-
 polkadot/node/core/pvf/tests/it/main.rs       |  13 +-
 .../node/core/pvf/tests/it/worker_common.rs   |  24 ++--
 polkadot/node/service/Cargo.toml              |   9 +-
 polkadot/node/service/src/workers.rs          |  15 +-
 polkadot/src/bin/execute-worker.rs            |   3 +-
 polkadot/src/bin/prepare-worker.rs            |   3 +-
 .../utils/build-script-utils/src/version.rs   |   6 +-
 16 files changed, 268 insertions(+), 64 deletions(-)
 create mode 100644 polkadot/node/core/pvf/benches/host_prepare_kusama_runtime.rs

diff --git a/.cargo/config.toml b/.cargo/config.toml
index 4796a2c26965..7fc466d3d3e9 100644
--- a/.cargo/config.toml
+++ b/.cargo/config.toml
@@ -31,3 +31,14 @@ rustflags = [
   "-Aclippy::stable_sort_primitive",     # prefer stable sort
   "-Aclippy::extra-unused-type-parameters", # stylistic
 ]
+
+[env]
+# The version of the node.
+#
+# This is the version that is used for versioning this node binary.
+# By default the `minor` version is bumped in every release. `Major` or `patch` releases are only
+# expected in very rare cases.
+#
+# The worker binaries associated to the node binary should ensure that they are using the same
+# version as the main node that started them.
+POLKADOT_NODE_VERSION = "1.1.0"
diff --git a/Cargo.lock b/Cargo.lock
index 683289adb45e..f2b6cc4bf10f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12022,9 +12022,11 @@ version = "1.0.0"
 dependencies = [
  "always-assert",
  "assert_matches",
+ "criterion 0.4.0",
  "futures",
  "futures-timer",
  "hex-literal",
+ "is_executable",
  "libc",
  "parity-scale-codec",
  "pin-project",
@@ -12042,7 +12044,7 @@ dependencies = [
  "sp-core",
  "sp-maybe-compressed-blob",
  "sp-wasm-interface",
- "substrate-build-script-utils",
+ "staging-kusama-runtime",
  "tempfile",
  "test-parachain-adder",
  "test-parachain-halt",
diff --git a/polkadot/cli/src/cli.rs b/polkadot/cli/src/cli.rs
index aaf8f1705760..0b1c948f79e4 100644
--- a/polkadot/cli/src/cli.rs
+++ b/polkadot/cli/src/cli.rs
@@ -19,16 +19,6 @@
 use clap::Parser;
 use std::path::PathBuf;
 
-/// The version of the node.
-///
-/// This is the version that is used for versioning this node binary.
-/// By default the `minor` version is bumped in every release. `Major` or `patch` releases are only
-/// expected in very rare cases.
-///
-/// The worker binaries associated to the node binary should ensure that they are using the same
-/// version as the main node that started them.
-pub const NODE_VERSION: &'static str = "1.1.0";
-
 #[allow(missing_docs)]
 #[derive(Debug, Parser)]
 pub enum Subcommand {
diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs
index 19437ce87530..77db9a316684 100644
--- a/polkadot/cli/src/command.rs
+++ b/polkadot/cli/src/command.rs
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::cli::{Cli, Subcommand, NODE_VERSION};
+use crate::cli::{Cli, Subcommand};
 use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE};
 use futures::future::TryFutureExt;
 use log::{info, warn};
@@ -50,7 +50,7 @@ impl SubstrateCli for Cli {
 
 	fn impl_version() -> String {
 		let commit_hash = env!("SUBSTRATE_CLI_COMMIT_HASH");
-		format!("{NODE_VERSION}-{commit_hash}")
+		format!("{}-{commit_hash}", env!("POLKADOT_NODE_VERSION"))
 	}
 
 	fn description() -> String {
@@ -262,8 +262,11 @@ where
 		None
 	};
 
-	let node_version =
-		if cli.run.disable_worker_version_check { None } else { Some(NODE_VERSION.to_string()) };
+	let node_version = if cli.run.disable_worker_version_check {
+		None
+	} else {
+		Some(env!("POLKADOT_NODE_VERSION").to_string())
+	};
 
 	runner.run_node_until_exit(move |config| async move {
 		let hwbench = (!cli.run.no_hardware_benchmarks)
diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml
index 478d1952d9d9..f79f98041ca4 100644
--- a/polkadot/node/core/pvf/Cargo.toml
+++ b/polkadot/node/core/pvf/Cargo.toml
@@ -11,6 +11,7 @@ always-assert = "0.1"
 futures = "0.3.21"
 futures-timer = "3.0.2"
 gum = { package = "tracing-gum", path = "../../gum" }
+is_executable = "1.0.1"
 libc = "0.2.139"
 pin-project = "1.0.9"
 rand = "0.8.5"
@@ -33,19 +34,22 @@ sp-maybe-compressed-blob = { path = "../../../../substrate/primitives/maybe-comp
 polkadot-node-core-pvf-prepare-worker = { path = "prepare-worker", optional = true }
 polkadot-node-core-pvf-execute-worker = { path = "execute-worker", optional = true }
 
-[build-dependencies]
-substrate-build-script-utils = { path = "../../../../substrate/utils/build-script-utils" }
-
 [dev-dependencies]
 assert_matches = "1.4.0"
+criterion = { version = "0.4.0", default-features = false, features = ["cargo_bench_support", "async_tokio"] }
 hex-literal = "0.4.1"
 polkadot-node-core-pvf-common = { path = "common", features = ["test-utils"] }
 # For the puppet worker, depend on ourselves with the test-utils feature.
 polkadot-node-core-pvf = { path = ".", features = ["test-utils"] }
+staging-kusama-runtime = { path = "../../../runtime/kusama" }
 
 adder = { package = "test-parachain-adder", path = "../../../parachain/test-parachains/adder" }
 halt = { package = "test-parachain-halt", path = "../../../parachain/test-parachains/halt" }
 
+[[bench]]
+name = "host_prepare_kusama_runtime"
+harness = false
+
 [features]
 ci-only-tests = []
 jemalloc-allocator = [ "polkadot-node-core-pvf-common/jemalloc-allocator" ]
diff --git a/polkadot/node/core/pvf/benches/host_prepare_kusama_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_kusama_runtime.rs
new file mode 100644
index 000000000000..8886bb84b7ff
--- /dev/null
+++ b/polkadot/node/core/pvf/benches/host_prepare_kusama_runtime.rs
@@ -0,0 +1,129 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+#[cfg(feature = "ci-only-tests")]
+use assert_matches::assert_matches;
+use criterion::{criterion_group, criterion_main, Criterion, SamplingMode};
+use polkadot_node_core_pvf::{
+	start, testing, Config, Metrics, PrepareError, PrepareJobKind, PrepareStats, PvfPrepData,
+	ValidationHost,
+};
+use polkadot_primitives::ExecutorParams;
+use std::{path::PathBuf, time::Duration};
+use tokio::{runtime::Handle, sync::Mutex};
+
+const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(30);
+
+struct TestHost {
+	cache_dir: PathBuf,
+	host: Mutex<ValidationHost>,
+}
+
+impl TestHost {
+	fn new_with_config<F>(handle: &Handle, f: F) -> Self
+	where
+		F: FnOnce(&mut Config),
+	{
+		let (prepare_worker_path, execute_worker_path) = testing::get_and_check_worker_paths();
+
+		let cache_dir = std::path::Path::new("/tmp/test").to_owned();
+		// let cache_dir = tempfile::tempdir().unwrap();
+		let mut config =
+			Config::new(cache_dir.clone(), None, prepare_worker_path, execute_worker_path);
+		f(&mut config);
+		let (host, task) = start(config, Metrics::default());
+		let _ = handle.spawn(task);
+		Self { cache_dir, host: Mutex::new(host) }
+	}
+
+	async fn precheck_pvf(
+		&self,
+		code: &[u8],
+		executor_params: ExecutorParams,
+	) -> Result<PrepareStats, PrepareError> {
+		let (result_tx, result_rx) = futures::channel::oneshot::channel();
+
+		let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024)
+			.expect("Compression works");
+
+		self.host
+			.lock()
+			.await
+			.precheck_pvf(
+				PvfPrepData::from_code(
+					code.into(),
+					executor_params,
+					TEST_PREPARATION_TIMEOUT,
+					PrepareJobKind::Prechecking,
+				),
+				result_tx,
+			)
+			.await
+			.unwrap();
+		result_rx.await.unwrap()
+	}
+}
+
+fn host_prepare_kusama_runtime(c: &mut Criterion) {
+	polkadot_node_core_pvf_common::sp_tracing::try_init_simple();
+
+	let rt = tokio::runtime::Runtime::new().unwrap();
+
+	let host = TestHost::new_with_config(rt.handle(), |cfg| {
+		cfg.prepare_workers_hard_max_num = 1;
+	});
+	let cache_dir = host.cache_dir.as_path();
+
+	let blob = staging_kusama_runtime::WASM_BINARY.unwrap();
+	let pvf = match sp_maybe_compressed_blob::decompress(&blob, 64 * 1024 * 1024) {
+		Ok(code) => PvfPrepData::from_code(
+			code.into_owned(),
+			ExecutorParams::default(),
+			Duration::from_secs(360),
+			PrepareJobKind::Compilation,
+		),
+		Err(e) => {
+			panic!("Cannot decompress blob: {:?}", e);
+		},
+	};
+
+	let mut group = c.benchmark_group("kusama");
+	group.sampling_mode(SamplingMode::Flat);
+	group.sample_size(20);
+	group.measurement_time(Duration::from_secs(240));
+	group.bench_function("host: prepare Kusama runtime", |b| {
+		b.to_async(&rt).iter(|| async {
+			// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
+			// benchmark accuracy
+			let _stats = host.precheck_pvf(&pvf.clone().code(), Default::default()).await.unwrap();
+
+			// Delete the prepared artifact. Otherwise the next iterations will immediately finish.
+			{
+				// Get the artifact path (asserting it exists).
+				let mut cache_dir: Vec<_> = std::fs::read_dir(cache_dir).unwrap().collect();
+				assert_eq!(cache_dir.len(), 1);
+				let artifact_path = cache_dir.pop().unwrap().unwrap();
+
+				// Delete the artifact.
+				std::fs::remove_file(artifact_path.path()).unwrap();
+			}
+		})
+	});
+	group.finish();
+}
+
+criterion_group!(preparation, host_prepare_kusama_runtime);
+criterion_main!(preparation);
diff --git a/polkadot/node/core/pvf/common/src/worker/mod.rs b/polkadot/node/core/pvf/common/src/worker/mod.rs
index bcdf882f300c..d50277ee3cc7 100644
--- a/polkadot/node/core/pvf/common/src/worker/mod.rs
+++ b/polkadot/node/core/pvf/common/src/worker/mod.rs
@@ -33,9 +33,13 @@ use tokio::{io, net::UnixStream, runtime::Runtime};
 /// spawning the desired worker.
 #[macro_export]
 macro_rules! decl_worker_main {
-	($expected_command:expr, $entrypoint:expr, $worker_version:expr $(,)*) => {
+	($expected_command:expr, $entrypoint:expr, $worker_version:expr, $worker_version_hash:expr $(,)*) => {
+		fn get_full_version() -> String {
+			format!("{}-{}", $worker_version, $worker_version_hash)
+		}
+
 		fn print_help(expected_command: &str) {
-			println!("{} {}", expected_command, $worker_version);
+			println!("{} {}", expected_command, get_full_version());
 			println!();
 			println!("PVF worker that is called by polkadot.");
 		}
@@ -60,6 +64,11 @@ macro_rules! decl_worker_main {
 					println!("{}", $worker_version);
 					return
 				},
+				// Useful for debugging. --version is used for version checks.
+				"--full-version" => {
+					println!("{}", get_full_version());
+					return
+				},
 				"test-sleep" => {
 					std::thread::sleep(std::time::Duration::from_secs(5));
 					return
diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs
index 0e4f2444adf7..80694059e38a 100644
--- a/polkadot/node/core/pvf/src/lib.rs
+++ b/polkadot/node/core/pvf/src/lib.rs
@@ -113,5 +113,18 @@ pub use polkadot_node_core_pvf_common::{
 	pvf::PvfPrepData,
 };
 
+use std::{path::Path, process::Command};
+
 /// The log target for this crate.
 pub const LOG_TARGET: &str = "parachain::pvf";
+
+/// Utility to get the version of a worker, used for version checks.
+///
+/// The worker's existence at the given path must be checked separately.
+pub fn get_worker_version(worker_path: &Path) -> std::io::Result<String> {
+	let worker_version = Command::new(worker_path).args(["--version"]).output()?.stdout;
+	Ok(std::str::from_utf8(&worker_version)
+		.expect("version is printed as a string; qed")
+		.trim()
+		.to_string())
+}
diff --git a/polkadot/node/core/pvf/src/testing.rs b/polkadot/node/core/pvf/src/testing.rs
index 4301afc3cc7e..97b0ca498dfc 100644
--- a/polkadot/node/core/pvf/src/testing.rs
+++ b/polkadot/node/core/pvf/src/testing.rs
@@ -15,13 +15,19 @@
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Various things for testing other crates.
-//!
-//! N.B. This is not guarded with some feature flag. Overexposing items here may affect the final
-//!      artifact even for production builds.
 
-pub use crate::worker_intf::{spawn_with_program_path, SpawnErr};
+pub use crate::{
+	host::{EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME},
+	worker_intf::{spawn_with_program_path, SpawnErr},
+};
 
+use crate::get_worker_version;
+use is_executable::IsExecutable;
 use polkadot_primitives::ExecutorParams;
+use std::{
+	path::PathBuf,
+	sync::{Mutex, OnceLock},
+};
 
 /// A function that emulates the stitches together behaviors of the preparation and the execution
 /// worker in a single synchronous function.
@@ -47,3 +53,47 @@ pub fn validate_candidate(
 
 	Ok(result)
 }
+
+/// Retrieves the worker paths, checks that they exist and does a version check.
+///
+/// NOTE: This should only be called in dev code (tests, benchmarks) as it relies on the relative
+/// paths of the built workers.
+pub fn get_and_check_worker_paths() -> (PathBuf, PathBuf) {
+	// Only needs to be called once for the current process.
+	static WORKER_PATHS: OnceLock<Mutex<(PathBuf, PathBuf)>> = OnceLock::new();
+	let mutex = WORKER_PATHS.get_or_init(|| {
+		let mut workers_path = std::env::current_exe().unwrap();
+		workers_path.pop();
+		workers_path.pop();
+		let mut prepare_worker_path = workers_path.clone();
+		prepare_worker_path.push(PREPARE_BINARY_NAME);
+		let mut execute_worker_path = workers_path.clone();
+		execute_worker_path.push(EXECUTE_BINARY_NAME);
+
+		// Check that the workers are valid.
+		if !prepare_worker_path.is_executable() || !execute_worker_path.is_executable() {
+			panic!("ERROR: Workers do not exist or are not executable. Workers directory: {:?}", workers_path);
+		}
+
+		let node_version = env!("POLKADOT_NODE_VERSION");
+		let worker_version =
+			get_worker_version(&prepare_worker_path).expect("checked for worker existence");
+		if worker_version != node_version {
+			panic!("ERROR: Prepare worker version {worker_version} does not match node version {node_version}; worker path: {prepare_worker_path:?}");
+		}
+		let worker_version =
+			get_worker_version(&execute_worker_path).expect("checked for worker existence");
+		if worker_version != node_version {
+			panic!("ERROR: Execute worker version {worker_version} does not match node version {node_version}; worker path: {execute_worker_path:?}");
+		}
+
+		// We don't want to check against the commit hash because we'd have to always rebuild
+		// the calling crate on every commit.
+		eprintln!("WARNING: Workers match the node version, but may have changed in recent commits. Please rebuild them if anything funny happens. Workers path: {workers_path:?}");
+
+		Mutex::new((prepare_worker_path, execute_worker_path))
+	});
+
+	let guard = mutex.lock().unwrap();
+	(guard.0.clone(), guard.1.clone())
+}
diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs
index dc8f00098ec5..e4c53a6d9240 100644
--- a/polkadot/node/core/pvf/tests/it/main.rs
+++ b/polkadot/node/core/pvf/tests/it/main.rs
@@ -18,8 +18,8 @@
 use assert_matches::assert_matches;
 use parity_scale_codec::Encode as _;
 use polkadot_node_core_pvf::{
-	start, Config, InvalidCandidate, Metrics, PrepareJobKind, PvfPrepData, ValidationError,
-	ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR,
+	start, testing::get_and_check_worker_paths, Config, InvalidCandidate, Metrics, PrepareJobKind,
+	PvfPrepData, ValidationError, ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR,
 };
 use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult};
 use polkadot_primitives::ExecutorParams;
@@ -50,13 +50,8 @@ impl TestHost {
 	where
 		F: FnOnce(&mut Config),
 	{
-		let mut workers_path = std::env::current_exe().unwrap();
-		workers_path.pop();
-		workers_path.pop();
-		let mut prepare_worker_path = workers_path.clone();
-		prepare_worker_path.push("polkadot-prepare-worker");
-		let mut execute_worker_path = workers_path.clone();
-		execute_worker_path.push("polkadot-execute-worker");
+		let (prepare_worker_path, execute_worker_path) = get_and_check_worker_paths();
+
 		let cache_dir = tempfile::tempdir().unwrap();
 		let mut config = Config::new(
 			cache_dir.path().to_owned(),
diff --git a/polkadot/node/core/pvf/tests/it/worker_common.rs b/polkadot/node/core/pvf/tests/it/worker_common.rs
index 875ae79af097..4640c976fb38 100644
--- a/polkadot/node/core/pvf/tests/it/worker_common.rs
+++ b/polkadot/node/core/pvf/tests/it/worker_common.rs
@@ -14,25 +14,21 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use polkadot_node_core_pvf::testing::{spawn_with_program_path, SpawnErr};
+use polkadot_node_core_pvf::testing::{
+	get_and_check_worker_paths, spawn_with_program_path, SpawnErr,
+};
 use std::time::Duration;
 
-fn worker_path(name: &str) -> std::path::PathBuf {
-	let mut worker_path = std::env::current_exe().unwrap();
-	worker_path.pop();
-	worker_path.pop();
-	worker_path.push(name);
-	worker_path
-}
-
 // Test spawning a program that immediately exits with a failure code.
 #[tokio::test]
 async fn spawn_immediate_exit() {
+	let (prepare_worker_path, _) = get_and_check_worker_paths();
+
 	// There's no explicit `exit` subcommand in the worker; it will panic on an unknown
 	// subcommand anyway
 	let result = spawn_with_program_path(
 		"integration-test",
-		worker_path("polkadot-prepare-worker"),
+		prepare_worker_path,
 		&["exit"],
 		Duration::from_secs(2),
 	)
@@ -42,9 +38,11 @@ async fn spawn_immediate_exit() {
 
 #[tokio::test]
 async fn spawn_timeout() {
+	let (_, execute_worker_path) = get_and_check_worker_paths();
+
 	let result = spawn_with_program_path(
 		"integration-test",
-		worker_path("polkadot-execute-worker"),
+		execute_worker_path,
 		&["test-sleep"],
 		Duration::from_secs(2),
 	)
@@ -54,9 +52,11 @@ async fn spawn_timeout() {
 
 #[tokio::test]
 async fn should_connect() {
+	let (prepare_worker_path, _) = get_and_check_worker_paths();
+
 	let _ = spawn_with_program_path(
 		"integration-test",
-		worker_path("polkadot-prepare-worker"),
+		prepare_worker_path,
 		&["prepare-worker"],
 		Duration::from_secs(2),
 	)
diff --git a/polkadot/node/service/Cargo.toml b/polkadot/node/service/Cargo.toml
index ee092e277332..6d36c3674028 100644
--- a/polkadot/node/service/Cargo.toml
+++ b/polkadot/node/service/Cargo.toml
@@ -74,9 +74,13 @@ frame-benchmarking-cli = { path = "../../../substrate/utils/frame/benchmarking-c
 frame-benchmarking = { path = "../../../substrate/frame/benchmarking" }
 
 # External Crates
+async-trait = "0.1.57"
 futures = "0.3.21"
 hex-literal = "0.4.1"
+is_executable = "1.0.1"
 gum = { package = "tracing-gum", path = "../gum" }
+log = "0.4.17"
+schnellru = "0.2.1"
 serde = { version = "1.0.188", features = ["derive"] }
 serde_json = "1.0.107"
 thiserror = "1.0.48"
@@ -85,11 +89,6 @@ kvdb-rocksdb = { version = "0.19.0", optional = true }
 parity-db = { version = "0.4.8", optional = true }
 codec = { package = "parity-scale-codec", version = "3.6.1" }
 
-async-trait = "0.1.57"
-schnellru = "0.2.1"
-log = "0.4.17"
-is_executable = "1.0.1"
-
 # Polkadot
 polkadot-core-primitives = { path = "../../core-primitives" }
 polkadot-node-core-parachains-inherent = { path = "../core/parachains-inherent" }
diff --git a/polkadot/node/service/src/workers.rs b/polkadot/node/service/src/workers.rs
index 5f7cc1c2ed49..defe73a9a105 100644
--- a/polkadot/node/service/src/workers.rs
+++ b/polkadot/node/service/src/workers.rs
@@ -18,7 +18,7 @@
 
 use super::Error;
 use is_executable::IsExecutable;
-use std::{path::PathBuf, process::Command};
+use std::path::PathBuf;
 
 #[cfg(test)]
 use std::sync::{Mutex, OnceLock};
@@ -75,11 +75,7 @@ pub fn determine_workers_paths(
 
 	// Do the version check.
 	if let Some(node_version) = node_version {
-		let worker_version = Command::new(&prep_worker_path).args(["--version"]).output()?.stdout;
-		let worker_version = std::str::from_utf8(&worker_version)
-			.expect("version is printed as a string; qed")
-			.trim()
-			.to_string();
+		let worker_version = polkadot_node_core_pvf::get_worker_version(&prep_worker_path)?;
 		if worker_version != node_version {
 			return Err(Error::WorkerBinaryVersionMismatch {
 				worker_version,
@@ -87,11 +83,8 @@ pub fn determine_workers_paths(
 				worker_path: prep_worker_path,
 			})
 		}
-		let worker_version = Command::new(&exec_worker_path).args(["--version"]).output()?.stdout;
-		let worker_version = std::str::from_utf8(&worker_version)
-			.expect("version is printed as a string; qed")
-			.trim()
-			.to_string();
+
+		let worker_version = polkadot_node_core_pvf::get_worker_version(&exec_worker_path)?;
 		if worker_version != node_version {
 			return Err(Error::WorkerBinaryVersionMismatch {
 				worker_version,
diff --git a/polkadot/src/bin/execute-worker.rs b/polkadot/src/bin/execute-worker.rs
index 1deb36580986..7fc7fe8514c3 100644
--- a/polkadot/src/bin/execute-worker.rs
+++ b/polkadot/src/bin/execute-worker.rs
@@ -19,5 +19,6 @@
 polkadot_node_core_pvf_common::decl_worker_main!(
 	"execute-worker",
 	polkadot_node_core_pvf_execute_worker::worker_entrypoint,
-	polkadot_cli::NODE_VERSION,
+	env!("POLKADOT_NODE_VERSION"),
+	env!("SUBSTRATE_CLI_COMMIT_HASH"),
 );
diff --git a/polkadot/src/bin/prepare-worker.rs b/polkadot/src/bin/prepare-worker.rs
index d731f8a30d06..3ea0bdf75ac0 100644
--- a/polkadot/src/bin/prepare-worker.rs
+++ b/polkadot/src/bin/prepare-worker.rs
@@ -19,5 +19,6 @@
 polkadot_node_core_pvf_common::decl_worker_main!(
 	"prepare-worker",
 	polkadot_node_core_pvf_prepare_worker::worker_entrypoint,
-	polkadot_cli::NODE_VERSION,
+	env!("POLKADOT_NODE_VERSION"),
+	env!("SUBSTRATE_CLI_COMMIT_HASH"),
 );
diff --git a/substrate/utils/build-script-utils/src/version.rs b/substrate/utils/build-script-utils/src/version.rs
index 309c6d71d77b..c2b2e23fe09f 100644
--- a/substrate/utils/build-script-utils/src/version.rs
+++ b/substrate/utils/build-script-utils/src/version.rs
@@ -31,7 +31,11 @@ pub fn generate_cargo_keys() {
 				Cow::from(sha)
 			},
 			Ok(o) => {
-				println!("cargo:warning=Git command failed with status: {}", o.status);
+				let stderr = String::from_utf8_lossy(&o.stderr).trim().to_owned();
+				println!(
+					"cargo:warning=Git command failed with status: '{}'; stderr: '{}'",
+					o.status, stderr,
+				);
 				Cow::from("unknown")
 			},
 			Err(err) => {

From ab2ea6c649ef2262633990ada2c0d4fdbb8aef2b Mon Sep 17 00:00:00 2001
From: Marcin S <marcin@realemail.net>
Date: Fri, 29 Sep 2023 15:25:26 +0200
Subject: [PATCH 02/11] Post-merge fixes

---
 Cargo.lock                                             |  4 +++-
 polkadot/node/core/pvf/Cargo.toml                      |  4 ++--
 ...usama_runtime.rs => host_prepare_rococo_runtime.rs} | 10 +++++-----
 3 files changed, 10 insertions(+), 8 deletions(-)
 rename polkadot/node/core/pvf/benches/{host_prepare_kusama_runtime.rs => host_prepare_rococo_runtime.rs} (93%)

diff --git a/Cargo.lock b/Cargo.lock
index e0ca0b012c64..9e8ebdc4df07 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -11935,9 +11935,11 @@ dependencies = [
  "always-assert",
  "assert_matches",
  "cfg-if",
+ "criterion 0.4.0",
  "futures",
  "futures-timer",
  "hex-literal",
+ "is_executable",
  "libc",
  "parity-scale-codec",
  "pin-project",
@@ -11951,11 +11953,11 @@ dependencies = [
  "polkadot-parachain-primitives",
  "polkadot-primitives",
  "rand 0.8.5",
+ "rococo-runtime",
  "slotmap",
  "sp-core",
  "sp-maybe-compressed-blob",
  "sp-wasm-interface",
- "substrate-build-script-utils",
  "tempfile",
  "test-parachain-adder",
  "test-parachain-halt",
diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml
index c56dfa80f086..809256ab46eb 100644
--- a/polkadot/node/core/pvf/Cargo.toml
+++ b/polkadot/node/core/pvf/Cargo.toml
@@ -42,13 +42,13 @@ hex-literal = "0.4.1"
 polkadot-node-core-pvf-common = { path = "common", features = ["test-utils"] }
 # For the puppet worker, depend on ourselves with the test-utils feature.
 polkadot-node-core-pvf = { path = ".", features = ["test-utils"] }
-staging-kusama-runtime = { path = "../../../runtime/kusama" }
+rococo-runtime = { path = "../../../runtime/rococo" }
 
 adder = { package = "test-parachain-adder", path = "../../../parachain/test-parachains/adder" }
 halt = { package = "test-parachain-halt", path = "../../../parachain/test-parachains/halt" }
 
 [[bench]]
-name = "host_prepare_kusama_runtime"
+name = "host_prepare_rococo_runtime"
 harness = false
 
 [features]
diff --git a/polkadot/node/core/pvf/benches/host_prepare_kusama_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
similarity index 93%
rename from polkadot/node/core/pvf/benches/host_prepare_kusama_runtime.rs
rename to polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
index 8886bb84b7ff..5141f363856c 100644
--- a/polkadot/node/core/pvf/benches/host_prepare_kusama_runtime.rs
+++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
@@ -77,7 +77,7 @@ impl TestHost {
 	}
 }
 
-fn host_prepare_kusama_runtime(c: &mut Criterion) {
+fn host_prepare_rococo_runtime(c: &mut Criterion) {
 	polkadot_node_core_pvf_common::sp_tracing::try_init_simple();
 
 	let rt = tokio::runtime::Runtime::new().unwrap();
@@ -87,7 +87,7 @@ fn host_prepare_kusama_runtime(c: &mut Criterion) {
 	});
 	let cache_dir = host.cache_dir.as_path();
 
-	let blob = staging_kusama_runtime::WASM_BINARY.unwrap();
+	let blob = rococo_runtime::WASM_BINARY.unwrap();
 	let pvf = match sp_maybe_compressed_blob::decompress(&blob, 64 * 1024 * 1024) {
 		Ok(code) => PvfPrepData::from_code(
 			code.into_owned(),
@@ -100,11 +100,11 @@ fn host_prepare_kusama_runtime(c: &mut Criterion) {
 		},
 	};
 
-	let mut group = c.benchmark_group("kusama");
+	let mut group = c.benchmark_group("rococo");
 	group.sampling_mode(SamplingMode::Flat);
 	group.sample_size(20);
 	group.measurement_time(Duration::from_secs(240));
-	group.bench_function("host: prepare Kusama runtime", |b| {
+	group.bench_function("host: prepare Rococo runtime", |b| {
 		b.to_async(&rt).iter(|| async {
 			// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
 			// benchmark accuracy
@@ -125,5 +125,5 @@ fn host_prepare_kusama_runtime(c: &mut Criterion) {
 	group.finish();
 }
 
-criterion_group!(preparation, host_prepare_kusama_runtime);
+criterion_group!(preparation, host_prepare_rococo_runtime);
 criterion_main!(preparation);

From 34363ec8818fc5c7f350b5f3d092fd55c2b44138 Mon Sep 17 00:00:00 2001
From: Marcin S <marcin@realemail.net>
Date: Mon, 2 Oct 2023 13:31:21 +0200
Subject: [PATCH 03/11] Fix artifact deletion in bench (add
 `prune_all_artifacts` test method)

---
 .../benches/host_prepare_rococo_runtime.rs    | 37 +++++++------
 .../node/core/pvf/common/src/worker/mod.rs    |  2 +-
 polkadot/node/core/pvf/src/artifacts.rs       |  1 +
 polkadot/node/core/pvf/src/host.rs            | 54 ++++++++++++++++---
 polkadot/node/core/pvf/src/worker_intf.rs     |  1 +
 polkadot/node/core/pvf/tests/it/main.rs       | 27 +++-------
 6 files changed, 77 insertions(+), 45 deletions(-)

diff --git a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
index 5141f363856c..ba993e959481 100644
--- a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
+++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
@@ -22,13 +22,14 @@ use polkadot_node_core_pvf::{
 	ValidationHost,
 };
 use polkadot_primitives::ExecutorParams;
-use std::{path::PathBuf, time::Duration};
+use std::time::Duration;
 use tokio::{runtime::Handle, sync::Mutex};
 
 const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(30);
 
 struct TestHost {
-	cache_dir: PathBuf,
+	#[allow(unused)]
+	cache_dir: tempfile::TempDir,
 	host: Mutex<ValidationHost>,
 }
 
@@ -39,10 +40,13 @@ impl TestHost {
 	{
 		let (prepare_worker_path, execute_worker_path) = testing::get_and_check_worker_paths();
 
-		let cache_dir = std::path::Path::new("/tmp/test").to_owned();
-		// let cache_dir = tempfile::tempdir().unwrap();
-		let mut config =
-			Config::new(cache_dir.clone(), None, prepare_worker_path, execute_worker_path);
+		let cache_dir = tempfile::tempdir().unwrap();
+		let mut config = Config::new(
+			cache_dir.path().to_owned(),
+			None,
+			prepare_worker_path,
+			execute_worker_path,
+		);
 		f(&mut config);
 		let (host, task) = start(config, Metrics::default());
 		let _ = handle.spawn(task);
@@ -75,6 +79,13 @@ impl TestHost {
 			.unwrap();
 		result_rx.await.unwrap()
 	}
+
+	async fn prune_all_artifacts(&self) -> Result<usize, std::io::Error> {
+		let (result_tx, result_rx) = futures::channel::oneshot::channel();
+
+		self.host.lock().await.prune_all_artifacts(result_tx).await.unwrap();
+		result_rx.await.unwrap()
+	}
 }
 
 fn host_prepare_rococo_runtime(c: &mut Criterion) {
@@ -85,7 +96,6 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 	let host = TestHost::new_with_config(rt.handle(), |cfg| {
 		cfg.prepare_workers_hard_max_num = 1;
 	});
-	let cache_dir = host.cache_dir.as_path();
 
 	let blob = rococo_runtime::WASM_BINARY.unwrap();
 	let pvf = match sp_maybe_compressed_blob::decompress(&blob, 64 * 1024 * 1024) {
@@ -104,6 +114,8 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 	group.sampling_mode(SamplingMode::Flat);
 	group.sample_size(20);
 	group.measurement_time(Duration::from_secs(240));
+	// The host spinning up a worker will be done in criterion's "warmup" stage, and not counted in
+	// the results.
 	group.bench_function("host: prepare Rococo runtime", |b| {
 		b.to_async(&rt).iter(|| async {
 			// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
@@ -111,15 +123,8 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 			let _stats = host.precheck_pvf(&pvf.clone().code(), Default::default()).await.unwrap();
 
 			// Delete the prepared artifact. Otherwise the next iterations will immediately finish.
-			{
-				// Get the artifact path (asserting it exists).
-				let mut cache_dir: Vec<_> = std::fs::read_dir(cache_dir).unwrap().collect();
-				assert_eq!(cache_dir.len(), 1);
-				let artifact_path = cache_dir.pop().unwrap().unwrap();
-
-				// Delete the artifact.
-				std::fs::remove_file(artifact_path.path()).unwrap();
-			}
+			let num_deleted = host.prune_all_artifacts().await.unwrap();
+			assert_eq!(num_deleted, 1);
 		})
 	});
 	group.finish();
diff --git a/polkadot/node/core/pvf/common/src/worker/mod.rs b/polkadot/node/core/pvf/common/src/worker/mod.rs
index 2b3ee51aaaf0..fd104f521f2f 100644
--- a/polkadot/node/core/pvf/common/src/worker/mod.rs
+++ b/polkadot/node/core/pvf/common/src/worker/mod.rs
@@ -42,7 +42,7 @@ macro_rules! decl_worker_main {
 
 		fn print_help(expected_command: &str) {
 			println!("{} {}", expected_command, $worker_version);
-			println!("commit: {}", worker_version_hash);
+			println!("commit: {}", $worker_version_hash);
 			println!();
 			println!("PVF worker that is called by polkadot.");
 		}
diff --git a/polkadot/node/core/pvf/src/artifacts.rs b/polkadot/node/core/pvf/src/artifacts.rs
index 5a1767af75b7..27111304598a 100644
--- a/polkadot/node/core/pvf/src/artifacts.rs
+++ b/polkadot/node/core/pvf/src/artifacts.rs
@@ -128,6 +128,7 @@ impl ArtifactPathId {
 	}
 }
 
+#[derive(Debug)]
 pub enum ArtifactState {
 	/// The artifact is ready to be used by the executor.
 	///
diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs
index 81695829122b..59729f8beef8 100644
--- a/polkadot/node/core/pvf/src/host.rs
+++ b/polkadot/node/core/pvf/src/host.rs
@@ -130,12 +130,33 @@ impl ValidationHost {
 			.await
 			.map_err(|_| "the inner loop hung up".to_string())
 	}
+
+	/// TEST-ONLY: resets the artifacts table.
+	#[cfg(feature = "test-utils")]
+	pub async fn prune_all_artifacts(
+		&mut self,
+		result_tx: oneshot::Sender<Result<usize, std::io::Error>>,
+	) -> Result<(), String> {
+		self.to_host_tx
+			.send(ToHost::PruneAllArtifacts { result_tx })
+			.await
+			.map_err(|_| "the inner loop hung up".to_string())
+	}
 }
 
 enum ToHost {
-	PrecheckPvf { pvf: PvfPrepData, result_tx: PrepareResultSender },
+	PrecheckPvf {
+		pvf: PvfPrepData,
+		result_tx: PrepareResultSender,
+	},
 	ExecutePvf(ExecutePvfInputs),
-	HeadsUp { active_pvfs: Vec<PvfPrepData> },
+	HeadsUp {
+		active_pvfs: Vec<PvfPrepData>,
+	},
+	#[cfg(feature = "test-utils")]
+	PruneAllArtifacts {
+		result_tx: oneshot::Sender<Result<usize, std::io::Error>>,
+	},
 }
 
 struct ExecutePvfInputs {
@@ -436,6 +457,25 @@ async fn handle_to_host(
 		},
 		ToHost::HeadsUp { active_pvfs } =>
 			handle_heads_up(artifacts, prepare_queue, active_pvfs).await?,
+		#[cfg(feature = "test-utils")]
+		ToHost::PruneAllArtifacts { result_tx } => {
+			let to_remove = artifacts.prune(Duration::ZERO);
+			gum::debug!(
+				target: LOG_TARGET,
+				"pruning all artifacts: {:?}",
+				to_remove
+			);
+			let mut result_to_send = Ok(to_remove.len());
+			for artifact_id in to_remove {
+				let artifact_path = artifact_id.path(cache_path);
+				let result = tokio::fs::remove_file(&artifact_path).await;
+				if let Err(err) = result {
+					result_to_send = Err(err);
+					break
+				}
+			}
+			let _ = result_tx.send(result_to_send);
+		},
 	}
 
 	Ok(())
@@ -446,7 +486,8 @@ async fn handle_to_host(
 /// This tries to prepare the PVF by compiling the WASM blob within a timeout set in
 /// `PvfPrepData`.
 ///
-/// If the prepare job failed previously, we may retry it under certain conditions.
+/// We don't retry artifacts that previously failed preparation. We don't expect multiple
+/// pre-checking requests.
 async fn handle_precheck_pvf(
 	artifacts: &mut Artifacts,
 	prepare_queue: &mut mpsc::Sender<prepare::ToQueue>,
@@ -464,8 +505,7 @@ async fn handle_precheck_pvf(
 			ArtifactState::Preparing { waiting_for_response, num_failures: _ } =>
 				waiting_for_response.push(result_sender),
 			ArtifactState::FailedToProcess { error, .. } => {
-				// Do not retry failed preparation if another pre-check request comes in. We do not
-				// retry pre-checking, anyway.
+				// Do not retry an artifact that previously failed preparation.
 				let _ = result_sender.send(PrepareResult::Err(error.clone()));
 			},
 		}
@@ -764,7 +804,7 @@ async fn handle_prepare_done(
 			let last_time_failed = SystemTime::now();
 			let num_failures = *num_failures + 1;
 
-			gum::warn!(
+			gum::error!(
 				target: LOG_TARGET,
 				?artifact_id,
 				time_failed = ?last_time_failed,
@@ -846,7 +886,7 @@ async fn sweeper_task(mut sweeper_rx: mpsc::Receiver<PathBuf>) {
 				gum::trace!(
 					target: LOG_TARGET,
 					?result,
-					"Sweeping the artifact file {}",
+					"Sweeped the artifact file {}",
 					condemned.display(),
 				);
 			},
diff --git a/polkadot/node/core/pvf/src/worker_intf.rs b/polkadot/node/core/pvf/src/worker_intf.rs
index 9825506ba88f..7a40f25cda00 100644
--- a/polkadot/node/core/pvf/src/worker_intf.rs
+++ b/polkadot/node/core/pvf/src/worker_intf.rs
@@ -255,6 +255,7 @@ impl WorkerHandle {
 		if let Ok(value) = std::env::var("RUST_LOG") {
 			command.env("RUST_LOG", value);
 		}
+
 		let mut child = command
 			.args(extra_args)
 			.arg("--worker-dir-path")
diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs
index be8a41181a52..cdf8d6eb82d2 100644
--- a/polkadot/node/core/pvf/tests/it/main.rs
+++ b/polkadot/node/core/pvf/tests/it/main.rs
@@ -18,8 +18,9 @@
 use assert_matches::assert_matches;
 use parity_scale_codec::Encode as _;
 use polkadot_node_core_pvf::{
-	start, testing::get_and_check_worker_paths, Config, InvalidCandidate, Metrics, PrepareJobKind,
-	PvfPrepData, ValidationError, ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR,
+	start, testing::get_and_check_worker_paths, Config, InvalidCandidate, Metrics, PrepareError,
+	PrepareJobKind, PrepareStats, PvfPrepData, ValidationError, ValidationHost,
+	JOB_TIMEOUT_WALL_CLOCK_FACTOR,
 };
 use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult};
 use polkadot_primitives::ExecutorParams;
@@ -291,25 +292,9 @@ async fn deleting_prepared_artifact_does_not_dispute() {
 	let host = TestHost::new();
 	let cache_dir = host.cache_dir.path();
 
-	let result = host
-		.validate_candidate(
-			halt::wasm_binary_unwrap(),
-			ValidationParams {
-				block_data: BlockData(Vec::new()),
-				parent_head: Default::default(),
-				relay_parent_number: 1,
-				relay_parent_storage_root: Default::default(),
-			},
-			Default::default(),
-		)
-		.await;
-
-	match result {
-		Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {},
-		r => panic!("{:?}", r),
-	}
+	let _stats = host.precheck_pvf(halt::wasm_binary_unwrap(), Default::default()).await.unwrap();
 
-	// Delete the prepared artifact.
+	// Manually delete the prepared artifact from disk. The in-memory artifacts table won't change.
 	{
 		// Get the artifact path (asserting it exists).
 		let mut cache_dir: Vec<_> = std::fs::read_dir(cache_dir).unwrap().collect();
@@ -324,7 +309,7 @@ async fn deleting_prepared_artifact_does_not_dispute() {
 		std::fs::remove_file(artifact_path.path()).unwrap();
 	}
 
-	// Try to validate again, artifact should get recreated.
+	// Try to validate, artifact should get recreated.
 	let result = host
 		.validate_candidate(
 			halt::wasm_binary_unwrap(),

From 07e2748d6fa1a20091cc4357c5426570948cef28 Mon Sep 17 00:00:00 2001
From: Marcin S <marcin@realemail.net>
Date: Mon, 2 Oct 2023 13:48:52 +0200
Subject: [PATCH 04/11] Improve log message a bit

---
 substrate/utils/build-script-utils/src/version.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/substrate/utils/build-script-utils/src/version.rs b/substrate/utils/build-script-utils/src/version.rs
index c2b2e23fe09f..f6a9ff9554ab 100644
--- a/substrate/utils/build-script-utils/src/version.rs
+++ b/substrate/utils/build-script-utils/src/version.rs
@@ -33,7 +33,7 @@ pub fn generate_cargo_keys() {
 			Ok(o) => {
 				let stderr = String::from_utf8_lossy(&o.stderr).trim().to_owned();
 				println!(
-					"cargo:warning=Git command failed with status: '{}'; stderr: '{}'",
+					"cargo:warning=Git command failed with status '{}' with message: '{}'",
 					o.status, stderr,
 				);
 				Cow::from("unknown")

From db4ca089aa8bd96017124800bdbd7270378f4629 Mon Sep 17 00:00:00 2001
From: Marcin S <marcin@realemail.net>
Date: Mon, 2 Oct 2023 13:54:42 +0200
Subject: [PATCH 05/11] Improve doc

---
 polkadot/node/core/pvf/src/host.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs
index 59729f8beef8..2da61df488e5 100644
--- a/polkadot/node/core/pvf/src/host.rs
+++ b/polkadot/node/core/pvf/src/host.rs
@@ -131,7 +131,7 @@ impl ValidationHost {
 			.map_err(|_| "the inner loop hung up".to_string())
 	}
 
-	/// TEST-ONLY: resets the artifacts table.
+	/// TEST-ONLY: deletes all on-disk artifacts and resets the artifacts table.
 	#[cfg(feature = "test-utils")]
 	pub async fn prune_all_artifacts(
 		&mut self,

From bbe8c5108fca694cd9381b4fd9ea56447c313568 Mon Sep 17 00:00:00 2001
From: Marcin S <marcin@realemail.net>
Date: Tue, 3 Oct 2023 11:31:42 +0200
Subject: [PATCH 06/11] Remove old import

---
 polkadot/tests/workers.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/polkadot/tests/workers.rs b/polkadot/tests/workers.rs
index 2872a1298dcd..960151415486 100644
--- a/polkadot/tests/workers.rs
+++ b/polkadot/tests/workers.rs
@@ -14,11 +14,11 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use polkadot_cli::NODE_VERSION;
 use std::process::Command;
 
 const PREPARE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-prepare-worker");
 const EXECUTE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-execute-worker");
+const POLKADOT_NODE_VERSION: &str = env!("POLKADOT_NODE_VERSION");
 
 #[test]
 fn worker_binaries_have_same_version_as_node() {
@@ -27,12 +27,12 @@ fn worker_binaries_have_same_version_as_node() {
 	let prep_worker_version = std::str::from_utf8(&prep_worker_version)
 		.expect("version is printed as a string; qed")
 		.trim();
-	assert_eq!(prep_worker_version, NODE_VERSION);
+	assert_eq!(prep_worker_version, POLKADOT_NODE_VERSION);
 
 	let exec_worker_version =
 		Command::new(&EXECUTE_WORKER_EXE).args(["--version"]).output().unwrap().stdout;
 	let exec_worker_version = std::str::from_utf8(&exec_worker_version)
 		.expect("version is printed as a string; qed")
 		.trim();
-	assert_eq!(exec_worker_version, NODE_VERSION);
+	assert_eq!(exec_worker_version, POLKADOT_NODE_VERSION);
 }

From 0412e70b043384ea49ef6f5a76f5b885f06a9c0a Mon Sep 17 00:00:00 2001
From: Marcin S <marcin@realemail.net>
Date: Fri, 6 Oct 2023 11:59:55 +0200
Subject: [PATCH 07/11] Address some review comments

---
 polkadot/node/core/pvf/Cargo.toml             |  3 +-
 .../benches/host_prepare_rococo_runtime.rs    | 46 ++++++++-----------
 polkadot/node/core/pvf/bin/puppet_worker.rs   | 17 -------
 polkadot/node/core/pvf/common/Cargo.toml      |  1 -
 polkadot/node/core/pvf/src/host.rs            | 44 +-----------------
 polkadot/node/service/src/workers.rs          |  4 +-
 6 files changed, 24 insertions(+), 91 deletions(-)
 delete mode 100644 polkadot/node/core/pvf/bin/puppet_worker.rs

diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml
index 809256ab46eb..bfd70c6fbd41 100644
--- a/polkadot/node/core/pvf/Cargo.toml
+++ b/polkadot/node/core/pvf/Cargo.toml
@@ -40,7 +40,8 @@ assert_matches = "1.4.0"
 criterion = { version = "0.4.0", default-features = false, features = ["cargo_bench_support", "async_tokio"] }
 hex-literal = "0.4.1"
 polkadot-node-core-pvf-common = { path = "common", features = ["test-utils"] }
-# For the puppet worker, depend on ourselves with the test-utils feature.
+# For benches and integration tests, depend on ourselves with the test-utils
+# feature.
 polkadot-node-core-pvf = { path = ".", features = ["test-utils"] }
 rococo-runtime = { path = "../../../runtime/rococo" }
 
diff --git a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
index ba993e959481..942730542218 100644
--- a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
+++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
@@ -14,9 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-#[cfg(feature = "ci-only-tests")]
-use assert_matches::assert_matches;
-use criterion::{criterion_group, criterion_main, Criterion, SamplingMode};
+use criterion::{criterion_group, criterion_main, BatchSize, Criterion, SamplingMode};
 use polkadot_node_core_pvf::{
 	start, testing, Config, Metrics, PrepareError, PrepareJobKind, PrepareStats, PvfPrepData,
 	ValidationHost,
@@ -28,8 +26,6 @@ use tokio::{runtime::Handle, sync::Mutex};
 const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(30);
 
 struct TestHost {
-	#[allow(unused)]
-	cache_dir: tempfile::TempDir,
 	host: Mutex<ValidationHost>,
 }
 
@@ -50,7 +46,7 @@ impl TestHost {
 		f(&mut config);
 		let (host, task) = start(config, Metrics::default());
 		let _ = handle.spawn(task);
-		Self { cache_dir, host: Mutex::new(host) }
+		Self { host: Mutex::new(host) }
 	}
 
 	async fn precheck_pvf(
@@ -79,13 +75,6 @@ impl TestHost {
 			.unwrap();
 		result_rx.await.unwrap()
 	}
-
-	async fn prune_all_artifacts(&self) -> Result<usize, std::io::Error> {
-		let (result_tx, result_rx) = futures::channel::oneshot::channel();
-
-		self.host.lock().await.prune_all_artifacts(result_tx).await.unwrap();
-		result_rx.await.unwrap()
-	}
 }
 
 fn host_prepare_rococo_runtime(c: &mut Criterion) {
@@ -93,10 +82,6 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 
 	let rt = tokio::runtime::Runtime::new().unwrap();
 
-	let host = TestHost::new_with_config(rt.handle(), |cfg| {
-		cfg.prepare_workers_hard_max_num = 1;
-	});
-
 	let blob = rococo_runtime::WASM_BINARY.unwrap();
 	let pvf = match sp_maybe_compressed_blob::decompress(&blob, 64 * 1024 * 1024) {
 		Ok(code) => PvfPrepData::from_code(
@@ -114,18 +99,23 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 	group.sampling_mode(SamplingMode::Flat);
 	group.sample_size(20);
 	group.measurement_time(Duration::from_secs(240));
-	// The host spinning up a worker will be done in criterion's "warmup" stage, and not counted in
-	// the results.
 	group.bench_function("host: prepare Rococo runtime", |b| {
-		b.to_async(&rt).iter(|| async {
-			// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
-			// benchmark accuracy
-			let _stats = host.precheck_pvf(&pvf.clone().code(), Default::default()).await.unwrap();
-
-			// Delete the prepared artifact. Otherwise the next iterations will immediately finish.
-			let num_deleted = host.prune_all_artifacts().await.unwrap();
-			assert_eq!(num_deleted, 1);
-		})
+		b.to_async(&rt).iter_batched(
+			|| {
+				(
+					TestHost::new_with_config(rt.handle(), |cfg| {
+						cfg.prepare_workers_hard_max_num = 1;
+					}),
+					pvf.clone().code(),
+				)
+			},
+			|(host, pvf_code)| async move {
+				// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
+				// benchmark accuracy
+				let _stats = host.precheck_pvf(&pvf_code, Default::default()).await.unwrap();
+			},
+			BatchSize::SmallInput,
+		)
 	});
 	group.finish();
 }
diff --git a/polkadot/node/core/pvf/bin/puppet_worker.rs b/polkadot/node/core/pvf/bin/puppet_worker.rs
deleted file mode 100644
index 7f93519d8454..000000000000
--- a/polkadot/node/core/pvf/bin/puppet_worker.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright (C) Parity Technologies (UK) Ltd.
-// This file is part of Polkadot.
-
-// Polkadot is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// Polkadot is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU General Public License for more details.
-
-// You should have received a copy of the GNU General Public License
-// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
-
-polkadot_node_core_pvf::decl_puppet_worker_main!();
diff --git a/polkadot/node/core/pvf/common/Cargo.toml b/polkadot/node/core/pvf/common/Cargo.toml
index 0f7308396d80..bceae7e49069 100644
--- a/polkadot/node/core/pvf/common/Cargo.toml
+++ b/polkadot/node/core/pvf/common/Cargo.toml
@@ -37,6 +37,5 @@ tempfile = "3.3.0"
 
 [features]
 # This feature is used to export test code to other crates without putting it in the production build.
-# Also used for building the puppet worker.
 test-utils = []
 jemalloc-allocator = []
diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs
index 2da61df488e5..6c9606bb2f3c 100644
--- a/polkadot/node/core/pvf/src/host.rs
+++ b/polkadot/node/core/pvf/src/host.rs
@@ -130,33 +130,12 @@ impl ValidationHost {
 			.await
 			.map_err(|_| "the inner loop hung up".to_string())
 	}
-
-	/// TEST-ONLY: deletes all on-disk artifacts and resets the artifacts table.
-	#[cfg(feature = "test-utils")]
-	pub async fn prune_all_artifacts(
-		&mut self,
-		result_tx: oneshot::Sender<Result<usize, std::io::Error>>,
-	) -> Result<(), String> {
-		self.to_host_tx
-			.send(ToHost::PruneAllArtifacts { result_tx })
-			.await
-			.map_err(|_| "the inner loop hung up".to_string())
-	}
 }
 
 enum ToHost {
-	PrecheckPvf {
-		pvf: PvfPrepData,
-		result_tx: PrepareResultSender,
-	},
+	PrecheckPvf { pvf: PvfPrepData, result_tx: PrepareResultSender },
 	ExecutePvf(ExecutePvfInputs),
-	HeadsUp {
-		active_pvfs: Vec<PvfPrepData>,
-	},
-	#[cfg(feature = "test-utils")]
-	PruneAllArtifacts {
-		result_tx: oneshot::Sender<Result<usize, std::io::Error>>,
-	},
+	HeadsUp { active_pvfs: Vec<PvfPrepData> },
 }
 
 struct ExecutePvfInputs {
@@ -457,25 +436,6 @@ async fn handle_to_host(
 		},
 		ToHost::HeadsUp { active_pvfs } =>
 			handle_heads_up(artifacts, prepare_queue, active_pvfs).await?,
-		#[cfg(feature = "test-utils")]
-		ToHost::PruneAllArtifacts { result_tx } => {
-			let to_remove = artifacts.prune(Duration::ZERO);
-			gum::debug!(
-				target: LOG_TARGET,
-				"pruning all artifacts: {:?}",
-				to_remove
-			);
-			let mut result_to_send = Ok(to_remove.len());
-			for artifact_id in to_remove {
-				let artifact_path = artifact_id.path(cache_path);
-				let result = tokio::fs::remove_file(&artifact_path).await;
-				if let Err(err) = result {
-					result_to_send = Err(err);
-					break
-				}
-			}
-			let _ = result_tx.send(result_to_send);
-		},
 	}
 
 	Ok(())
diff --git a/polkadot/node/service/src/workers.rs b/polkadot/node/service/src/workers.rs
index defe73a9a105..e8cd9e4c65ed 100644
--- a/polkadot/node/service/src/workers.rs
+++ b/polkadot/node/service/src/workers.rs
@@ -305,7 +305,7 @@ echo {}
 			);
 
 			// Try with valid provided workers path that is a binary file.
-			let given_workers_path = tempdir.join("usr/local/bin/puppet-worker");
+			let given_workers_path = tempdir.join("usr/local/bin/test-worker");
 			write_worker_exe(&given_workers_path)?;
 			assert_matches!(
 				determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())),
@@ -461,7 +461,7 @@ echo {}
 			);
 
 			// Given worker binary returns bad version.
-			let given_workers_path = tempdir.join("usr/local/bin/puppet-worker");
+			let given_workers_path = tempdir.join("usr/local/bin/test-worker");
 			write_worker_exe_invalid_version(&given_workers_path, bad_version)?;
 			assert_matches!(
 				determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())),

From 8b830234b029778fea8db6951320f36a55401599 Mon Sep 17 00:00:00 2001
From: "Marcin S." <marcin@realemail.net>
Date: Wed, 18 Oct 2023 10:30:16 +0200
Subject: [PATCH 08/11] Add execute benchmark

---
 polkadot/node/core/pvf/Cargo.toml             |   2 +-
 ...coco_runtime.rs => host_rococo_runtime.rs} | 106 +++++++++++++++++-
 2 files changed, 102 insertions(+), 6 deletions(-)
 rename polkadot/node/core/pvf/benches/{host_prepare_rococo_runtime.rs => host_rococo_runtime.rs} (54%)

diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml
index bfd70c6fbd41..5034f7d0b742 100644
--- a/polkadot/node/core/pvf/Cargo.toml
+++ b/polkadot/node/core/pvf/Cargo.toml
@@ -49,7 +49,7 @@ adder = { package = "test-parachain-adder", path = "../../../parachain/test-para
 halt = { package = "test-parachain-halt", path = "../../../parachain/test-parachains/halt" }
 
 [[bench]]
-name = "host_prepare_rococo_runtime"
+name = "host_rococo_runtime"
 harness = false
 
 [features]
diff --git a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_rococo_runtime.rs
similarity index 54%
rename from polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
rename to polkadot/node/core/pvf/benches/host_rococo_runtime.rs
index 942730542218..38a144a3b5c0 100644
--- a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
+++ b/polkadot/node/core/pvf/benches/host_rococo_runtime.rs
@@ -14,15 +14,26 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
+//! Benchmarks for preparation and execution through the host.
+//!
+//! # Examples
+//!
+//! $ cargo bench
+//! $ cargo bench prepare
+//! $ cargo bench execute
+
 use criterion::{criterion_group, criterion_main, BatchSize, Criterion, SamplingMode};
+use parity_scale_codec::Encode;
 use polkadot_node_core_pvf::{
 	start, testing, Config, Metrics, PrepareError, PrepareJobKind, PrepareStats, PvfPrepData,
-	ValidationHost,
+	ValidationError, ValidationHost,
 };
+use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult};
 use polkadot_primitives::ExecutorParams;
 use std::time::Duration;
 use tokio::{runtime::Handle, sync::Mutex};
 
+const TEST_EXECUTION_TIMEOUT: Duration = Duration::from_secs(3);
 const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(30);
 
 struct TestHost {
@@ -75,6 +86,37 @@ impl TestHost {
 			.unwrap();
 		result_rx.await.unwrap()
 	}
+
+	async fn validate_candidate(
+		&self,
+		code: &[u8],
+		params: ValidationParams,
+		executor_params: ExecutorParams,
+	) -> Result<ValidationResult, ValidationError> {
+		let (result_tx, result_rx) = futures::channel::oneshot::channel();
+
+		let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024)
+			.expect("Compression works");
+
+		self.host
+			.lock()
+			.await
+			.execute_pvf(
+				PvfPrepData::from_code(
+					code.into(),
+					executor_params,
+					TEST_PREPARATION_TIMEOUT,
+					PrepareJobKind::Compilation,
+				),
+				TEST_EXECUTION_TIMEOUT,
+				params.encode(),
+				polkadot_node_core_pvf::Priority::Normal,
+				result_tx,
+			)
+			.await
+			.unwrap();
+		result_rx.await.unwrap()
+	}
 }
 
 fn host_prepare_rococo_runtime(c: &mut Criterion) {
@@ -95,7 +137,7 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 		},
 	};
 
-	let mut group = c.benchmark_group("rococo");
+	let mut group = c.benchmark_group("prepare rococo");
 	group.sampling_mode(SamplingMode::Flat);
 	group.sample_size(20);
 	group.measurement_time(Duration::from_secs(240));
@@ -111,7 +153,7 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 			},
 			|(host, pvf_code)| async move {
 				// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
-				// benchmark accuracy
+				// benchmark accuracy.
 				let _stats = host.precheck_pvf(&pvf_code, Default::default()).await.unwrap();
 			},
 			BatchSize::SmallInput,
@@ -120,5 +162,59 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 	group.finish();
 }
 
-criterion_group!(preparation, host_prepare_rococo_runtime);
-criterion_main!(preparation);
+fn host_execute_rococo_runtime(c: &mut Criterion) {
+	polkadot_node_core_pvf_common::sp_tracing::try_init_simple();
+
+	let rt = tokio::runtime::Runtime::new().unwrap();
+
+	let host = TestHost::new_with_config(rt.handle(), |cfg| {
+		cfg.execute_workers_max_num = 1;
+	});
+
+	let blob = rococo_runtime::WASM_BINARY.unwrap();
+	let pvf = match sp_maybe_compressed_blob::decompress(&blob, 64 * 1024 * 1024) {
+		Ok(code) => PvfPrepData::from_code(
+			code.into_owned(),
+			ExecutorParams::default(),
+			Duration::from_secs(360),
+			PrepareJobKind::Compilation,
+		),
+		Err(e) => {
+			panic!("Cannot decompress blob: {:?}", e);
+		},
+	};
+
+	// Prepare beforehand, so that the benchmark only measures execution.
+	rt.block_on(async {
+		let _stats = host.precheck_pvf(&pvf.code(), Default::default()).await.unwrap();
+	});
+
+	let mut group = c.benchmark_group("execute rococo");
+	group.sampling_mode(SamplingMode::Flat);
+	group.sample_size(20);
+	group.measurement_time(Duration::from_secs(240));
+	group.bench_function("host: prepare Rococo runtime", |b| {
+		b.to_async(&rt).iter(|| async {
+			// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
+			// benchmark accuracy.
+			let _result = host
+				.validate_candidate(
+					&pvf.code(),
+					ValidationParams {
+						block_data: BlockData(Vec::new()),
+						parent_head: Default::default(),
+						relay_parent_number: 1,
+						relay_parent_storage_root: Default::default(),
+					},
+					Default::default(),
+				)
+				.await
+				.unwrap();
+		})
+	});
+	group.finish();
+}
+
+criterion_group!(prepare, host_prepare_rococo_runtime);
+criterion_group!(execute, host_execute_rococo_runtime);
+criterion_main!(prepare, execute);

From 185df8381b4ac8a44143aa17c91928a135fca87b Mon Sep 17 00:00:00 2001
From: "Marcin S." <marcin@realemail.net>
Date: Thu, 19 Oct 2023 12:57:23 +0200
Subject: [PATCH 09/11] Use NODE_VERSION constant

---
 polkadot/cli/src/command.rs           | 11 +++----
 polkadot/node/core/pvf/src/testing.rs | 10 +++----
 polkadot/node/service/src/workers.rs  | 42 +++++++++++++--------------
 polkadot/src/bin/execute-worker.rs    |  2 +-
 polkadot/src/bin/prepare-worker.rs    |  2 +-
 polkadot/tests/workers.rs             |  6 ++--
 6 files changed, 35 insertions(+), 38 deletions(-)

diff --git a/polkadot/cli/src/command.rs b/polkadot/cli/src/command.rs
index 84ed3ed93670..2dcf5e0e8d7b 100644
--- a/polkadot/cli/src/command.rs
+++ b/polkadot/cli/src/command.rs
@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-use crate::cli::{Cli, Subcommand};
+use crate::cli::{Cli, Subcommand, NODE_VERSION};
 use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE};
 use futures::future::TryFutureExt;
 use log::{info, warn};
@@ -50,7 +50,7 @@ impl SubstrateCli for Cli {
 
 	fn impl_version() -> String {
 		let commit_hash = env!("SUBSTRATE_CLI_COMMIT_HASH");
-		format!("{}-{commit_hash}", env!("POLKADOT_NODE_VERSION"))
+		format!("{}-{commit_hash}", NODE_VERSION)
 	}
 
 	fn description() -> String {
@@ -241,11 +241,8 @@ where
 		None
 	};
 
-	let node_version = if cli.run.disable_worker_version_check {
-		None
-	} else {
-		Some(env!("POLKADOT_NODE_VERSION").to_string())
-	};
+	let node_version =
+		if cli.run.disable_worker_version_check { None } else { Some(NODE_VERSION.to_string()) };
 
 	runner.run_node_until_exit(move |config| async move {
 		let hwbench = (!cli.run.no_hardware_benchmarks)
diff --git a/polkadot/node/core/pvf/src/testing.rs b/polkadot/node/core/pvf/src/testing.rs
index 723fd0537e6e..31bcfe7fadb6 100644
--- a/polkadot/node/core/pvf/src/testing.rs
+++ b/polkadot/node/core/pvf/src/testing.rs
@@ -23,6 +23,7 @@ pub use crate::{
 
 use crate::get_worker_version;
 use is_executable::IsExecutable;
+use polkadot_node_primitives::NODE_VERSION;
 use polkadot_primitives::ExecutorParams;
 use std::{
 	path::PathBuf,
@@ -75,16 +76,15 @@ pub fn get_and_check_worker_paths() -> (PathBuf, PathBuf) {
 			panic!("ERROR: Workers do not exist or are not executable. Workers directory: {:?}", workers_path);
 		}
 
-		let node_version = env!("POLKADOT_NODE_VERSION");
 		let worker_version =
 			get_worker_version(&prepare_worker_path).expect("checked for worker existence");
-		if worker_version != node_version {
-			panic!("ERROR: Prepare worker version {worker_version} does not match node version {node_version}; worker path: {prepare_worker_path:?}");
+		if worker_version != NODE_VERSION {
+			panic!("ERROR: Prepare worker version {worker_version} does not match node version {NODE_VERSION}; worker path: {prepare_worker_path:?}");
 		}
 		let worker_version =
 			get_worker_version(&execute_worker_path).expect("checked for worker existence");
-		if worker_version != node_version {
-			panic!("ERROR: Execute worker version {worker_version} does not match node version {node_version}; worker path: {execute_worker_path:?}");
+		if worker_version != NODE_VERSION {
+			panic!("ERROR: Execute worker version {worker_version} does not match node version {NODE_VERSION}; worker path: {execute_worker_path:?}");
 		}
 
 		// We don't want to check against the commit hash because we'd have to always rebuild
diff --git a/polkadot/node/service/src/workers.rs b/polkadot/node/service/src/workers.rs
index e8cd9e4c65ed..b35bb8302fdc 100644
--- a/polkadot/node/service/src/workers.rs
+++ b/polkadot/node/service/src/workers.rs
@@ -208,11 +208,11 @@ mod tests {
 	use serial_test::serial;
 	use std::{env::temp_dir, fs, os::unix::fs::PermissionsExt, path::Path};
 
-	const NODE_VERSION: &'static str = "v0.1.2";
+	const TEST_NODE_VERSION: &'static str = "v0.1.2";
 
 	/// Write a dummy executable to the path which satisfies the version check.
 	fn write_worker_exe(path: impl AsRef<Path>) -> Result<(), Box<dyn std::error::Error>> {
-		let program = get_program(NODE_VERSION);
+		let program = get_program(TEST_NODE_VERSION);
 		fs::write(&path, program)?;
 		Ok(fs::set_permissions(&path, fs::Permissions::from_mode(0o744))?)
 	}
@@ -280,7 +280,7 @@ echo {}
 
 			// Try with provided workers path that has missing binaries.
 			assert_matches!(
-				determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())),
+				determine_workers_paths(Some(given_workers_path.clone()), None, Some(TEST_NODE_VERSION.into())),
 				Err(Error::MissingWorkerBinaries { given_workers_path: Some(p1), current_exe_path: p2, workers_names: None }) if p1 == given_workers_path && p2 == exe_path
 			);
 
@@ -292,7 +292,7 @@ echo {}
 			write_worker_exe(&execute_worker_path)?;
 			fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o644))?;
 			assert_matches!(
-				determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())),
+				determine_workers_paths(Some(given_workers_path.clone()), None, Some(TEST_NODE_VERSION.into())),
 				Err(Error::InvalidWorkerBinaries { prep_worker_path: p1, exec_worker_path: p2 }) if p1 == prepare_worker_path && p2 == execute_worker_path
 			);
 
@@ -300,7 +300,7 @@ echo {}
 			fs::set_permissions(&prepare_worker_path, fs::Permissions::from_mode(0o744))?;
 			fs::set_permissions(&execute_worker_path, fs::Permissions::from_mode(0o744))?;
 			assert_matches!(
-				determine_workers_paths(Some(given_workers_path), None, Some(NODE_VERSION.into())),
+				determine_workers_paths(Some(given_workers_path), None, Some(TEST_NODE_VERSION.into())),
 				Ok((p1, p2)) if p1 == prepare_worker_path && p2 == execute_worker_path
 			);
 
@@ -308,7 +308,7 @@ echo {}
 			let given_workers_path = tempdir.join("usr/local/bin/test-worker");
 			write_worker_exe(&given_workers_path)?;
 			assert_matches!(
-				determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())),
+				determine_workers_paths(Some(given_workers_path.clone()), None, Some(TEST_NODE_VERSION.into())),
 				Ok((p1, p2)) if p1 == given_workers_path && p2 == given_workers_path
 			);
 
@@ -323,7 +323,7 @@ echo {}
 		with_temp_dir_structure(|tempdir, exe_path| {
 			// Try with both binaries missing.
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
 				Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path
 			);
 
@@ -331,7 +331,7 @@ echo {}
 			let prepare_worker_path = tempdir.join("usr/bin/polkadot-prepare-worker");
 			write_worker_exe(&prepare_worker_path)?;
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
 				Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path
 			);
 
@@ -340,7 +340,7 @@ echo {}
 			let execute_worker_path = tempdir.join("usr/bin/polkadot-execute-worker");
 			write_worker_exe(&execute_worker_path)?;
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
 				Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path
 			);
 
@@ -349,7 +349,7 @@ echo {}
 			let prepare_worker_path = tempdir.join("usr/lib/polkadot/polkadot-prepare-worker");
 			write_worker_exe(&prepare_worker_path)?;
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
 				Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path
 			);
 
@@ -358,7 +358,7 @@ echo {}
 			let execute_worker_path = tempdir.join("usr/lib/polkadot/polkadot-execute-worker");
 			write_worker_exe(execute_worker_path)?;
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
 				Err(Error::MissingWorkerBinaries { given_workers_path: None, current_exe_path: p, workers_names: None }) if p == exe_path
 			);
 
@@ -433,8 +433,8 @@ echo {}
 			write_worker_exe_invalid_version(&prepare_worker_bin_path, bad_version)?;
 			write_worker_exe(&execute_worker_bin_path)?;
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
-				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == prepare_worker_bin_path
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
+				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == TEST_NODE_VERSION && p == prepare_worker_bin_path
 			);
 
 			// Workers at lib location return bad version.
@@ -445,8 +445,8 @@ echo {}
 			write_worker_exe(&prepare_worker_lib_path)?;
 			write_worker_exe_invalid_version(&execute_worker_lib_path, bad_version)?;
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
-				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == execute_worker_lib_path
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
+				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == TEST_NODE_VERSION && p == execute_worker_lib_path
 			);
 
 			// Workers at provided workers location return bad version.
@@ -456,16 +456,16 @@ echo {}
 			write_worker_exe_invalid_version(&prepare_worker_path, bad_version)?;
 			write_worker_exe_invalid_version(&execute_worker_path, bad_version)?;
 			assert_matches!(
-				determine_workers_paths(Some(given_workers_path), None, Some(NODE_VERSION.into())),
-				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == prepare_worker_path
+				determine_workers_paths(Some(given_workers_path), None, Some(TEST_NODE_VERSION.into())),
+				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == TEST_NODE_VERSION && p == prepare_worker_path
 			);
 
 			// Given worker binary returns bad version.
 			let given_workers_path = tempdir.join("usr/local/bin/test-worker");
 			write_worker_exe_invalid_version(&given_workers_path, bad_version)?;
 			assert_matches!(
-				determine_workers_paths(Some(given_workers_path.clone()), None, Some(NODE_VERSION.into())),
-				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == NODE_VERSION && p == given_workers_path
+				determine_workers_paths(Some(given_workers_path.clone()), None, Some(TEST_NODE_VERSION.into())),
+				Err(Error::WorkerBinaryVersionMismatch { worker_version: v1, node_version: v2, worker_path: p }) if v1 == bad_version && v2 == TEST_NODE_VERSION && p == given_workers_path
 			);
 
 			Ok(())
@@ -485,7 +485,7 @@ echo {}
 			write_worker_exe(&execute_worker_bin_path)?;
 
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
 				Ok((p1, p2)) if p1 == prepare_worker_bin_path && p2 == execute_worker_bin_path
 			);
 
@@ -502,7 +502,7 @@ echo {}
 			write_worker_exe(&execute_worker_lib_path)?;
 
 			assert_matches!(
-				determine_workers_paths(None, None, Some(NODE_VERSION.into())),
+				determine_workers_paths(None, None, Some(TEST_NODE_VERSION.into())),
 				Ok((p1, p2)) if p1 == prepare_worker_lib_path && p2 == execute_worker_lib_path
 			);
 
diff --git a/polkadot/src/bin/execute-worker.rs b/polkadot/src/bin/execute-worker.rs
index 7fc7fe8514c3..c39a5a3c89de 100644
--- a/polkadot/src/bin/execute-worker.rs
+++ b/polkadot/src/bin/execute-worker.rs
@@ -19,6 +19,6 @@
 polkadot_node_core_pvf_common::decl_worker_main!(
 	"execute-worker",
 	polkadot_node_core_pvf_execute_worker::worker_entrypoint,
-	env!("POLKADOT_NODE_VERSION"),
+	polkadot_cli::NODE_VERSION,
 	env!("SUBSTRATE_CLI_COMMIT_HASH"),
 );
diff --git a/polkadot/src/bin/prepare-worker.rs b/polkadot/src/bin/prepare-worker.rs
index 3ea0bdf75ac0..3b42991f18ba 100644
--- a/polkadot/src/bin/prepare-worker.rs
+++ b/polkadot/src/bin/prepare-worker.rs
@@ -19,6 +19,6 @@
 polkadot_node_core_pvf_common::decl_worker_main!(
 	"prepare-worker",
 	polkadot_node_core_pvf_prepare_worker::worker_entrypoint,
-	env!("POLKADOT_NODE_VERSION"),
+	polkadot_cli::NODE_VERSION,
 	env!("SUBSTRATE_CLI_COMMIT_HASH"),
 );
diff --git a/polkadot/tests/workers.rs b/polkadot/tests/workers.rs
index 960151415486..2872a1298dcd 100644
--- a/polkadot/tests/workers.rs
+++ b/polkadot/tests/workers.rs
@@ -14,11 +14,11 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
+use polkadot_cli::NODE_VERSION;
 use std::process::Command;
 
 const PREPARE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-prepare-worker");
 const EXECUTE_WORKER_EXE: &str = env!("CARGO_BIN_EXE_polkadot-execute-worker");
-const POLKADOT_NODE_VERSION: &str = env!("POLKADOT_NODE_VERSION");
 
 #[test]
 fn worker_binaries_have_same_version_as_node() {
@@ -27,12 +27,12 @@ fn worker_binaries_have_same_version_as_node() {
 	let prep_worker_version = std::str::from_utf8(&prep_worker_version)
 		.expect("version is printed as a string; qed")
 		.trim();
-	assert_eq!(prep_worker_version, POLKADOT_NODE_VERSION);
+	assert_eq!(prep_worker_version, NODE_VERSION);
 
 	let exec_worker_version =
 		Command::new(&EXECUTE_WORKER_EXE).args(["--version"]).output().unwrap().stdout;
 	let exec_worker_version = std::str::from_utf8(&exec_worker_version)
 		.expect("version is printed as a string; qed")
 		.trim();
-	assert_eq!(exec_worker_version, POLKADOT_NODE_VERSION);
+	assert_eq!(exec_worker_version, NODE_VERSION);
 }

From 7644bb6aba5f312a178079d867f523b8ba3d9f8b Mon Sep 17 00:00:00 2001
From: "Marcin S." <marcin@realemail.net>
Date: Mon, 23 Oct 2023 15:43:24 +0200
Subject: [PATCH 10/11] Remove bad execute benchmark

---
 polkadot/node/core/pvf/Cargo.toml             |  2 +-
 ...time.rs => host_prepare_rococo_runtime.rs} | 98 +------------------
 2 files changed, 5 insertions(+), 95 deletions(-)
 rename polkadot/node/core/pvf/benches/{host_rococo_runtime.rs => host_prepare_rococo_runtime.rs} (59%)

diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml
index 5034f7d0b742..bfd70c6fbd41 100644
--- a/polkadot/node/core/pvf/Cargo.toml
+++ b/polkadot/node/core/pvf/Cargo.toml
@@ -49,7 +49,7 @@ adder = { package = "test-parachain-adder", path = "../../../parachain/test-para
 halt = { package = "test-parachain-halt", path = "../../../parachain/test-parachains/halt" }
 
 [[bench]]
-name = "host_rococo_runtime"
+name = "host_prepare_rococo_runtime"
 harness = false
 
 [features]
diff --git a/polkadot/node/core/pvf/benches/host_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
similarity index 59%
rename from polkadot/node/core/pvf/benches/host_rococo_runtime.rs
rename to polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
index 38a144a3b5c0..3069fa2b194b 100644
--- a/polkadot/node/core/pvf/benches/host_rococo_runtime.rs
+++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs
@@ -14,13 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Benchmarks for preparation and execution through the host.
-//!
-//! # Examples
-//!
-//! $ cargo bench
-//! $ cargo bench prepare
-//! $ cargo bench execute
+//! Benchmarks for preparation through the host. We use a real PVF to get realistic results.
 
 use criterion::{criterion_group, criterion_main, BatchSize, Criterion, SamplingMode};
 use parity_scale_codec::Encode;
@@ -30,6 +24,7 @@ use polkadot_node_core_pvf::{
 };
 use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult};
 use polkadot_primitives::ExecutorParams;
+use rococo_runtime::WASM_BINARY;
 use std::time::Duration;
 use tokio::{runtime::Handle, sync::Mutex};
 
@@ -86,37 +81,6 @@ impl TestHost {
 			.unwrap();
 		result_rx.await.unwrap()
 	}
-
-	async fn validate_candidate(
-		&self,
-		code: &[u8],
-		params: ValidationParams,
-		executor_params: ExecutorParams,
-	) -> Result<ValidationResult, ValidationError> {
-		let (result_tx, result_rx) = futures::channel::oneshot::channel();
-
-		let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024)
-			.expect("Compression works");
-
-		self.host
-			.lock()
-			.await
-			.execute_pvf(
-				PvfPrepData::from_code(
-					code.into(),
-					executor_params,
-					TEST_PREPARATION_TIMEOUT,
-					PrepareJobKind::Compilation,
-				),
-				TEST_EXECUTION_TIMEOUT,
-				params.encode(),
-				polkadot_node_core_pvf::Priority::Normal,
-				result_tx,
-			)
-			.await
-			.unwrap();
-		result_rx.await.unwrap()
-	}
 }
 
 fn host_prepare_rococo_runtime(c: &mut Criterion) {
@@ -124,7 +88,7 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 
 	let rt = tokio::runtime::Runtime::new().unwrap();
 
-	let blob = rococo_runtime::WASM_BINARY.unwrap();
+	let blob = WASM_BINARY.expect("You need to build the WASM binaries to run the tests!");
 	let pvf = match sp_maybe_compressed_blob::decompress(&blob, 64 * 1024 * 1024) {
 		Ok(code) => PvfPrepData::from_code(
 			code.into_owned(),
@@ -162,59 +126,5 @@ fn host_prepare_rococo_runtime(c: &mut Criterion) {
 	group.finish();
 }
 
-fn host_execute_rococo_runtime(c: &mut Criterion) {
-	polkadot_node_core_pvf_common::sp_tracing::try_init_simple();
-
-	let rt = tokio::runtime::Runtime::new().unwrap();
-
-	let host = TestHost::new_with_config(rt.handle(), |cfg| {
-		cfg.execute_workers_max_num = 1;
-	});
-
-	let blob = rococo_runtime::WASM_BINARY.unwrap();
-	let pvf = match sp_maybe_compressed_blob::decompress(&blob, 64 * 1024 * 1024) {
-		Ok(code) => PvfPrepData::from_code(
-			code.into_owned(),
-			ExecutorParams::default(),
-			Duration::from_secs(360),
-			PrepareJobKind::Compilation,
-		),
-		Err(e) => {
-			panic!("Cannot decompress blob: {:?}", e);
-		},
-	};
-
-	// Prepare beforehand, so that the benchmark only measures execution.
-	rt.block_on(async {
-		let _stats = host.precheck_pvf(&pvf.code(), Default::default()).await.unwrap();
-	});
-
-	let mut group = c.benchmark_group("execute rococo");
-	group.sampling_mode(SamplingMode::Flat);
-	group.sample_size(20);
-	group.measurement_time(Duration::from_secs(240));
-	group.bench_function("host: prepare Rococo runtime", |b| {
-		b.to_async(&rt).iter(|| async {
-			// `PvfPrepData` is designed to be cheap to clone, so cloning shouldn't affect the
-			// benchmark accuracy.
-			let _result = host
-				.validate_candidate(
-					&pvf.code(),
-					ValidationParams {
-						block_data: BlockData(Vec::new()),
-						parent_head: Default::default(),
-						relay_parent_number: 1,
-						relay_parent_storage_root: Default::default(),
-					},
-					Default::default(),
-				)
-				.await
-				.unwrap();
-		})
-	});
-	group.finish();
-}
-
 criterion_group!(prepare, host_prepare_rococo_runtime);
-criterion_group!(execute, host_execute_rococo_runtime);
-criterion_main!(prepare, execute);
+criterion_main!(prepare);

From 9abb1665861dc7d957bc3ea641191168241889ed Mon Sep 17 00:00:00 2001
From: "Marcin S." <marcin@realemail.net>
Date: Tue, 24 Oct 2023 15:19:27 +0200
Subject: [PATCH 11/11] Update Cargo.lock

---
 Cargo.lock | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Cargo.lock b/Cargo.lock
index 25f86121b945..d1a1b1877a92 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12914,6 +12914,7 @@ dependencies = [
  "sc-telemetry",
  "sc-transaction-pool",
  "sc-transaction-pool-api",
+ "schnellru",
  "serde",
  "serde_json",
  "serial_test",