Merge pull request #2520 from subspace/prometheus-metrics-refactoring

Prometheus metrics refactoring

crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs

@@ -107,8 +107,8 @@ pub(crate) struct FarmingArgs {
     no_info: bool,
     /// Defines endpoints for the prometheus metrics server. It doesn't start without at least
     /// one specified endpoint. Format: 127.0.0.1:8080
-    #[arg(long, alias = "metrics-endpoint")]
-    metrics_endpoints: Vec<SocketAddr>,
+    #[arg(long, aliases = ["metrics-endpoint", "metrics-endpoints"])]
+    prometheus_listen_on: Vec<SocketAddr>,
     /// Defines how many sectors farmer will download concurrently, allows to limit memory usage of
     /// the plotting process, defaults to `--sector-encoding-concurrency` + 1 to download future
     /// sector ahead of time
@@ -314,7 +314,7 @@ where
         dev,
         tmp,
         mut disk_farms,
-        metrics_endpoints,
+        prometheus_listen_on,
         sector_downloading_concurrency,
         sector_encoding_concurrency,
         farm_during_initial_plotting,
@@ -384,7 +384,7 @@ where
     // Metrics
     let mut prometheus_metrics_registry = Registry::default();
     let farmer_metrics = FarmerMetrics::new(&mut prometheus_metrics_registry);
-    let metrics_endpoints_are_specified = !metrics_endpoints.is_empty();
+    let should_start_prometheus_server = !prometheus_listen_on.is_empty();
 
     let (node, mut node_runner) = {
         if dsn.bootstrap_nodes.is_empty() {
@@ -399,14 +399,14 @@ where
             Arc::downgrade(&readers_and_pieces),
             node_client.clone(),
             piece_cache.clone(),
-            metrics_endpoints_are_specified.then_some(&mut prometheus_metrics_registry),
+            should_start_prometheus_server.then_some(&mut prometheus_metrics_registry),
         )?
     };
 
-    let _prometheus_worker = if metrics_endpoints_are_specified {
+    let _prometheus_worker = if should_start_prometheus_server {
         let prometheus_task = start_prometheus_metrics_server(
-            metrics_endpoints,
-            RegistryAdapter::Libp2p(prometheus_metrics_registry),
+            prometheus_listen_on,
+            RegistryAdapter::PrometheusClient(prometheus_metrics_registry),
         )?;
 
         let join_handle = tokio::spawn(prometheus_task);

crates/subspace-malicious-operator/src/bin/subspace-malicious-operator.rs

@@ -227,6 +227,7 @@ fn main() -> Result<(), Error> {
             let consensus_chain_node = subspace_service::new_full::<PosTable, _>(
                 consensus_chain_config,
                 partial_components,
+                None,
                 true,
                 SlotProportion::new(3f32 / 4f32),
             )

crates/subspace-networking/examples/metrics.rs

@@ -32,7 +32,7 @@ async fn main() {
 
     match start_prometheus_metrics_server(
         vec![prometheus_metrics_server_address],
-        RegistryAdapter::Libp2p(metric_registry),
+        RegistryAdapter::PrometheusClient(metric_registry),
     ) {
         Err(err) => {
             error!(

crates/subspace-networking/src/bin/subspace-bootstrap-node/main.rs

@@ -85,8 +85,8 @@ enum Command {
         external_addresses: Vec<Multiaddr>,
         /// Defines endpoints for the prometheus metrics server. It doesn't start without at least
         /// one specified endpoint. Format: 127.0.0.1:8080
-        #[arg(long, alias = "metrics-endpoint")]
-        metrics_endpoints: Vec<SocketAddr>,
+        #[arg(long, aliases = ["metrics-endpoint", "metrics-endpoints"])]
+        prometheus_listen_on: Vec<SocketAddr>,
     },
     /// Generate a new keypair
     GenerateKeypair {
@@ -149,7 +149,7 @@ async fn main() -> Result<(), Box<dyn Error>> {
             allow_private_ips,
             protocol_version,
             external_addresses,
-            metrics_endpoints,
+            prometheus_listen_on,
         } => {
             debug!(
                 "Libp2p protocol stack instantiated with version: {} ",
@@ -160,10 +160,10 @@ async fn main() -> Result<(), Box<dyn Error>> {
             let keypair = identity::Keypair::from(decoded_keypair);
 
             // Metrics
-            let metrics_endpoints_are_specified = !metrics_endpoints.is_empty();
+            let should_start_prometheus_server = !prometheus_listen_on.is_empty();
             let mut metrics_registry = Registry::default();
             let dsn_metrics_registry =
-                metrics_endpoints_are_specified.then_some(&mut metrics_registry);
+                should_start_prometheus_server.then_some(&mut metrics_registry);
 
             let known_peers_registry_config = KnownPeersManagerConfig {
                 enable_known_peers_source: false,
@@ -217,11 +217,11 @@ async fn main() -> Result<(), Box<dyn Error>> {
 
             info!("Subspace Bootstrap Node started");
 
-            let prometheus_task = metrics_endpoints_are_specified
+            let prometheus_task = should_start_prometheus_server
                 .then(|| {
                     start_prometheus_metrics_server(
-                        metrics_endpoints,
-                        RegistryAdapter::Libp2p(metrics_registry),
+                        prometheus_listen_on,
+                        RegistryAdapter::PrometheusClient(metrics_registry),
                     )
                 })
                 .transpose()?;

crates/subspace-node/Cargo.toml

@@ -40,6 +40,7 @@ hex = "0.4.3"
 hex-literal = "0.4.1"
 mimalloc = "0.1.39"
 parity-scale-codec = "3.6.9"
+prometheus-client = "0.22.0"
 sc-chain-spec = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 sc-cli = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false }
 sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
@@ -66,11 +67,13 @@ sp-keystore = { version = "0.27.0", git = "https://github.com/subspace/polkadot-
 sp-messenger = { version = "0.1.0", path = "../../domains/primitives/messenger" }
 sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" }
+subspace-metrics = { version = "0.1.0", path = "../../shared/subspace-metrics" }
 subspace-networking = { version = "0.1.0", path = "../subspace-networking" }
 subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" }
 subspace-runtime = { version = "0.1.0", path = "../subspace-runtime" }
 subspace-runtime-primitives = { version = "0.1.0", path = "../subspace-runtime-primitives" }
 subspace-service = { version = "0.1.0", path = "../subspace-service" }
+substrate-prometheus-endpoint = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" }
 supports-color = "2.1.0"
 tempfile = "3.9.0"
 thiserror = "1.0.56"

crates/subspace-node/src/commands/run.rs

@@ -23,6 +23,7 @@ use sc_utils::mpsc::tracing_unbounded;
 use sp_core::traits::SpawnEssentialNamed;
 use sp_messenger::messages::ChainId;
 use std::env;
+use subspace_metrics::{start_prometheus_metrics_server, RegistryAdapter};
 use subspace_runtime::{Block, RuntimeApi};
 use tracing::{debug, error, info, info_span, warn};
 
@@ -86,6 +87,7 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> {
         dev,
         pot_external_entropy,
         storage_monitor,
+        mut prometheus_configuration,
     } = create_consensus_chain_configuration(consensus, enable_color, domain_options.is_some())?;
 
     let maybe_domain_configuration = domain_options
@@ -130,6 +132,11 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> {
             let full_node_fut = subspace_service::new_full::<PosTable, _>(
                 subspace_configuration,
                 partial_components,
+                prometheus_configuration
+                    .as_mut()
+                    .map(|prometheus_configuration| {
+                        &mut prometheus_configuration.prometheus_registry
+                    }),
                 true,
                 SlotProportion::new(3f32 / 4f32),
             );
@@ -274,6 +281,26 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> {
 
         consensus_chain_node.network_starter.start_network();
 
+        if let Some(prometheus_configuration) = prometheus_configuration.take() {
+            let metrics_server = start_prometheus_metrics_server(
+                vec![prometheus_configuration.listen_on],
+                RegistryAdapter::Both(
+                    prometheus_configuration.prometheus_registry,
+                    prometheus_configuration.substrate_registry,
+                ),
+            )
+            .map_err(|error| Error::SubspaceService(error.into()))?
+            .map(|error| {
+                debug!(?error, "Metrics server error.");
+            });
+
+            consensus_chain_node.task_manager.spawn_handle().spawn(
+                "metrics-server",
+                None,
+                metrics_server,
+            );
+        };
+
         consensus_chain_node.task_manager
     };
 

crates/subspace-node/src/commands/run/consensus.rs

@@ -1,6 +1,7 @@
 use crate::commands::run::shared::RpcOptions;
 use crate::{chain_spec, derive_pot_external_entropy, Error};
 use clap::Parser;
+use prometheus_client::registry::Registry;
 use sc_chain_spec::GenericChainSpec;
 use sc_cli::{
     generate_node_name, Cors, NodeKeyParams, NodeKeyType, RpcMethods, TelemetryParams,
@@ -408,13 +409,20 @@ pub(super) struct ConsensusChainOptions {
     timekeeper_options: TimekeeperOptions,
 }
 
+pub(super) struct PrometheusConfiguration {
+    pub(super) listen_on: SocketAddr,
+    pub(super) prometheus_registry: Registry,
+    pub(super) substrate_registry: substrate_prometheus_endpoint::Registry,
+}
+
 pub(super) struct ConsensusChainConfiguration {
     pub(super) maybe_tmp_dir: Option<TempDir>,
     pub(super) subspace_configuration: SubspaceConfiguration,
     pub(super) dev: bool,
     /// External entropy, used initially when PoT chain starts to derive the first seed
     pub(super) pot_external_entropy: Vec<u8>,
     pub(super) storage_monitor: StorageMonitorParams,
+    pub(super) prometheus_configuration: Option<PrometheusConfiguration>,
 }
 
 pub(super) fn create_consensus_chain_configuration(
@@ -640,6 +648,7 @@ pub(super) fn create_consensus_chain_configuration(
         }
     };
 
+    let substrate_registry = consensus_chain_config.prometheus_registry().cloned();
     Ok(ConsensusChainConfiguration {
         maybe_tmp_dir,
         subspace_configuration: SubspaceConfiguration {
@@ -655,5 +664,12 @@ pub(super) fn create_consensus_chain_configuration(
         dev,
         pot_external_entropy,
         storage_monitor,
+        prometheus_configuration: prometheus_listen_on.zip(substrate_registry).map(
+            |(listen_on, substrate_registry)| PrometheusConfiguration {
+                listen_on,
+                prometheus_registry: Registry::default(),
+                substrate_registry,
+            },
+        ),
     })
 }

crates/subspace-service/Cargo.toml

@@ -73,7 +73,6 @@ sp-transaction-pool = { version = "4.0.0-dev", git = "https://github.com/subspac
 static_assertions = "1.1.0"
 subspace-archiving = { version = "0.1.0", path = "../subspace-archiving" }
 subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives" }
-subspace-metrics = { version = "0.1.0", path = "../../shared/subspace-metrics" }
 subspace-networking = { version = "0.1.0", path = "../subspace-networking" }
 subspace-proof-of-space = { version = "0.1.0", path = "../subspace-proof-of-space" }
 subspace-runtime-primitives = { version = "0.1.0", path = "../subspace-runtime-primitives" }

crates/subspace-service/src/config.rs

@@ -1,6 +1,5 @@
 use crate::dsn::DsnConfig;
 use crate::sync_from_dsn::DsnSyncPieceGetter;
-use prometheus_client::registry::Registry;
 use sc_chain_spec::ChainSpec;
 use sc_network::config::{
     MultiaddrWithPeerId, NetworkConfiguration, NodeKeyConfig, SetConfig, SyncMode, TransportConfig,
@@ -223,8 +222,6 @@ pub enum SubspaceNetworking {
         node: Node,
         /// Bootstrap nodes used (that can be also sent to the farmer over RPC)
         bootstrap_nodes: Vec<Multiaddr>,
-        /// DSN metrics registry
-        metrics_registry: Option<Registry>,
     },
     /// Networking must be instantiated internally
     Create {

crates/subspace-service/src/dsn.rs

@@ -71,13 +71,10 @@ pub struct DsnConfig {
 pub(crate) fn create_dsn_instance(
     dsn_protocol_version: String,
     dsn_config: DsnConfig,
-    enable_metrics: bool,
-) -> Result<(Node, NodeRunner<()>, Option<Registry>), DsnConfigurationError> {
+    prometheus_registry: Option<&mut Registry>,
+) -> Result<(Node, NodeRunner<()>), DsnConfigurationError> {
     trace!("Subspace networking starting.");
 
-    let mut metrics_registry = Registry::default();
-    let dsn_metrics_registry = enable_metrics.then_some(&mut metrics_registry);
-
     let networking_parameters_registry = {
         let network_path = dsn_config.network_path;
 
@@ -101,7 +98,7 @@ pub(crate) fn create_dsn_instance(
 
     let keypair = dsn_config.keypair.clone();
     let default_networking_config =
-        subspace_networking::Config::new(dsn_protocol_version, keypair, (), dsn_metrics_registry);
+        subspace_networking::Config::new(dsn_protocol_version, keypair, (), prometheus_registry);
 
     let networking_config = subspace_networking::Config {
         keypair: dsn_config.keypair.clone(),
@@ -126,13 +123,5 @@ pub(crate) fn create_dsn_instance(
         ..default_networking_config
     };
 
-    subspace_networking::construct(networking_config)
-        .map(|(node, node_runner)| {
-            (
-                node,
-                node_runner,
-                enable_metrics.then_some(metrics_registry),
-            )
-        })
-        .map_err(Into::into)
+    subspace_networking::construct(networking_config).map_err(Into::into)
 }