From a8f5ccb1e3e7fdf4561ed33234e0ed0ddaaef94e Mon Sep 17 00:00:00 2001 From: Liu-Cheng Xu Date: Sun, 7 Jul 2024 16:02:32 +0800 Subject: [PATCH] Integrate storage monitor (#5) * Integrate storage monitor * Fix clippy --- Cargo.lock | 45 ++++++++++++++++---- Cargo.toml | 2 + crates/subcoin-node/Cargo.toml | 1 + crates/subcoin-node/src/cli.rs | 22 +++++++++- crates/subcoin-service/Cargo.toml | 3 ++ crates/subcoin-service/src/lib.rs | 68 ++++++++++++++++++++++++++++--- 6 files changed, 125 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 899a995b33c21..b0268cdeab251 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2714,6 +2714,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs4" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f9df8a11882c4e3335eb2d18a0137c505d9ca927470b0cac9c6f0ae07d28f7" +dependencies = [ + "rustix 0.38.34", + "windows-sys 0.48.0", +] + [[package]] name = "funty" version = "2.0.0" @@ -7339,6 +7349,19 @@ dependencies = [ "sp-core", ] +[[package]] +name = "sc-storage-monitor" +version = "0.16.0" +source = "git+https://github.com/liuchengxu/polkadot-sdk?branch=subcoin#c4a04f05a2d1c5e463385e74014d36a867d73adf" +dependencies = [ + "clap", + "fs4", + "log", + "sp-core", + "thiserror", + "tokio", +] + [[package]] name = "sc-sysinfo" version = "27.0.0" @@ -8203,7 +8226,7 @@ dependencies = [ [[package]] name = "sp-crypto-ec-utils" version = "0.10.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "ark-bls12-377", "ark-bls12-377-ext", @@ -8265,7 +8288,7 @@ dependencies = [ [[package]] name = "sp-debug-derive" version = "14.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "proc-macro2", "quote", @@ -8285,7 +8308,7 @@ dependencies = [ [[package]] name = "sp-externalities" version = "0.25.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "environmental", "parity-scale-codec", @@ -8472,7 +8495,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "24.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "bytes", "impl-trait-for-tuples", @@ -8504,7 +8527,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" version = "17.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "Inflector", "expander", @@ -8593,7 +8616,7 @@ source = "git+https://github.com/liuchengxu/polkadot-sdk?branch=subcoin#c4a04f05 [[package]] name = "sp-std" version = "14.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" [[package]] name = "sp-storage" @@ -8610,7 +8633,7 @@ dependencies = [ [[package]] name = "sp-storage" version = "19.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "impl-serde", "parity-scale-codec", @@ -8645,7 +8668,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "16.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "parity-scale-codec", "tracing", @@ -8742,7 +8765,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" version = "20.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#b6f18232449b003d716a0d630f3da3a9dce669ab" +source = "git+https://github.com/paritytech/polkadot-sdk#221eddc90cd1efc4fc3c822ce5ccf289272fb41d" dependencies = [ "impl-trait-for-tuples", "log", @@ -8956,6 +8979,7 @@ dependencies = [ "sc-network-sync", "sc-rpc", "sc-service", + "sc-storage-monitor", "sc-telemetry", "sc-transaction-pool", "sc-transaction-pool-api", @@ -9023,6 +9047,7 @@ version = "0.1.0" dependencies = [ "async-trait", "bitcoin 0.32.2", + "frame-benchmarking-cli", "frame-system", "futures", "jsonrpsee", @@ -9034,6 +9059,8 @@ dependencies = [ "sc-network", "sc-rpc", "sc-service", + "sc-storage-monitor", + "sc-sysinfo", "sc-telemetry", "sc-transaction-pool", "sc-utils", diff --git a/Cargo.toml b/Cargo.toml index 7b5b7f187faf4..aacb45ccb0366 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,8 @@ sc-network-sync = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = sc-rpc = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = "subcoin" } # Disable the default `rocksdb` feature sc-service = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = "subcoin", default-features = false } +sc-storage-monitor = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = "subcoin" } +sc-sysinfo = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = "subcoin" } sc-telemetry = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = "subcoin" } sc-transaction-pool = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = "subcoin" } sc-transaction-pool-api = { git = "https://github.com/liuchengxu/polkadot-sdk", branch = "subcoin" } diff --git a/crates/subcoin-node/Cargo.toml b/crates/subcoin-node/Cargo.toml index 204f1a2315b9c..48d96b687ea56 100644 --- a/crates/subcoin-node/Cargo.toml +++ b/crates/subcoin-node/Cargo.toml @@ -32,6 +32,7 @@ sc-network = { workspace = true } sc-network-sync = { workspace = true } sc-rpc = { workspace = true } sc-service = { workspace = true } +sc-storage-monitor = { workspace = true } sc-telemetry = { workspace = true } sc-transaction-pool = { workspace = true } sc-transaction-pool-api = { workspace = true } diff --git a/crates/subcoin-node/src/cli.rs b/crates/subcoin-node/src/cli.rs index 7174ec2885878..ed639e44a9a60 100644 --- a/crates/subcoin-node/src/cli.rs +++ b/crates/subcoin-node/src/cli.rs @@ -40,11 +40,29 @@ pub enum Command { pub struct Cli { #[command(subcommand)] pub command: Command, + + /// Disable automatic hardware benchmarks. + /// + /// By default these benchmarks are automatically ran at startup and measure + /// the CPU speed, the memory bandwidth and the disk speed. + /// + /// The results are then printed out in the logs, and also sent as part of + /// telemetry, if telemetry is enabled. + #[arg(long)] + pub no_hardware_benchmarks: bool, + + #[allow(missing_docs)] + #[clap(flatten)] + pub storage_monitor: sc_storage_monitor::StorageMonitorParams, } /// Parse and run command line arguments pub fn run() -> sc_cli::Result<()> { - let Cli { command } = Cli::parse(); + let Cli { + command, + no_hardware_benchmarks, + storage_monitor, + } = Cli::parse(); match command { Command::ImportBlocks(cmd) => { @@ -59,6 +77,8 @@ pub fn run() -> sc_cli::Result<()> { } = subcoin_service::new_node(subcoin_service::SubcoinConfiguration { network: bitcoin::Network::Bitcoin, config: &config, + no_hardware_benchmarks, + storage_monitor, })?; task_manager.spawn_handle().spawn("finalizer", None, { let client = client.clone(); diff --git a/crates/subcoin-service/Cargo.toml b/crates/subcoin-service/Cargo.toml index d667c93fec7c1..50756feedc676 100644 --- a/crates/subcoin-service/Cargo.toml +++ b/crates/subcoin-service/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true [dependencies] async-trait = { workspace = true } bitcoin = { workspace = true } +frame-benchmarking-cli = { workspace = true } frame-system = { workspace = true } futures = { workspace = true } jsonrpsee = { workspace = true } @@ -21,6 +22,8 @@ sc-network = { workspace = true } sc-rpc = { workspace = true } # `test-helpers` for the exposed Client type. sc-service = { workspace = true, features = ["test-helpers"], default-features = false } +sc-storage-monitor = { workspace = true } +sc-sysinfo = { workspace = true } sc-telemetry = { workspace = true } sc-transaction-pool = { workspace = true } sc-utils = { workspace = true } diff --git a/crates/subcoin-service/src/lib.rs b/crates/subcoin-service/src/lib.rs index 14bed437fbf94..240e53f9af590 100644 --- a/crates/subcoin-service/src/lib.rs +++ b/crates/subcoin-service/src/lib.rs @@ -1,3 +1,5 @@ +//! Subcoin service implementation. Specialized wrapper over substrate service. + #![allow(deprecated)] pub mod chain_spec; @@ -5,6 +7,7 @@ mod genesis_block_builder; mod transaction_adapter; use bitcoin::hashes::Hash; +use frame_benchmarking_cli::SUBSTRATE_REFERENCE_HARDWARE; use futures::StreamExt; use genesis_block_builder::GenesisBlockBuilder; use sc_client_api::{AuxStore, BlockchainEvents, Finalizer, HeaderBackend}; @@ -21,6 +24,7 @@ use sp_core::Encode; use sp_keystore::KeystorePtr; use sp_runtime::traits::{Block as BlockT, CheckedSub}; use std::ops::Deref; +use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use subcoin_runtime::interface::OpaqueBlock as Block; @@ -135,7 +139,7 @@ async fn build_system_rpc_future( } } -/// Bitcoin node components. +/// Subcoin node components. pub struct NodeComponents { /// Client. pub client: Arc, @@ -149,10 +153,12 @@ pub struct NodeComponents { pub system_rpc_tx: TracingUnboundedSender>, } -/// Substrate configuration plus some subcoin specific configs. +/// Subcoin node configuration. pub struct SubcoinConfiguration<'a> { pub network: bitcoin::Network, pub config: &'a Configuration, + pub no_hardware_benchmarks: bool, + pub storage_monitor: sc_storage_monitor::StorageMonitorParams, } impl<'a> Deref for SubcoinConfiguration<'a> { @@ -183,6 +189,13 @@ fn initialize_genesis_block_hash_mapping Result { + let SubcoinConfiguration { + network: bitcoin_network, + config, + no_hardware_benchmarks, + storage_monitor, + } = config; + let telemetry = config .telemetry_endpoints .clone() @@ -195,12 +208,10 @@ pub fn new_node(config: SubcoinConfiguration) -> Result::new( bitcoin_network, config.chain_spec.as_storage_builder(), @@ -211,7 +222,7 @@ pub fn new_node(config: SubcoinConfiguration) -> Result( - config.deref(), + config, telemetry.as_ref().map(|(_, telemetry)| telemetry.handle()), executor.clone(), backend, @@ -224,6 +235,13 @@ pub fn new_node(config: SubcoinConfiguration) -> Result Result { + tracing::warn!( + "⚠️ The hardware does not meet the minimal requirements {err} for role 'Authority'.", + ); + } + _ => {} + } + + if let Some(ref mut telemetry) = telemetry { + let telemetry_handle = telemetry.handle(); + task_manager.spawn_handle().spawn( + "telemetry_hwbench", + None, + sc_sysinfo::initialize_hwbench_telemetry(telemetry_handle, hwbench), + ); + } + } + + if let Some(database_path) = database_path { + sc_storage_monitor::StorageMonitorService::try_spawn( + storage_monitor, + database_path, + &task_manager.spawn_essential_handle(), + ) + .map_err(|e| ServiceError::Application(e.into()))?; + } + Ok(NodeComponents { client, backend,