feat: prometheus metrics for RPC methods

CHANGELOG.md

@@ -32,6 +32,9 @@
 - [#3959](https://github.com/ChainSafe/forest/issues/3959) Added support for the
   Ethereum RPC name aliases.
 
+- [#4607](https://github.com/ChainSafe/forest/pull/4607) Expose usage and timing
+  metrics for RPC methods.
+
 ### Changed
 
 ### Removed

src/metrics/mod.rs

@@ -9,7 +9,11 @@ use once_cell::sync::Lazy;
 use parking_lot::{RwLock, RwLockWriteGuard};
 use prometheus_client::{
     encoding::EncodeLabelSet,
-    metrics::{counter::Counter, family::Family, histogram::Histogram},
+    metrics::{
+        counter::Counter,
+        family::Family,
+        histogram::{exponential_buckets, Histogram},
+    },
 };
 use std::sync::Arc;
 use std::{path::PathBuf, time::Instant};
@@ -38,6 +42,29 @@ pub static LRU_CACHE_MISS: Lazy<Family<KindLabel, Counter>> = Lazy::new(|| {
     metric
 });
 
+pub static RPC_METHOD_FAILURE: Lazy<Family<RpcMethodLabel, Counter>> = Lazy::new(|| {
+    let metric = Family::default();
+    DEFAULT_REGISTRY.write().register(
+        "rpc_method_failure",
+        "Number of failed RPC calls",
+        metric.clone(),
+    );
+    metric
+});
+
+pub static RPC_METHOD_TIME: Lazy<Family<RpcMethodLabel, Histogram>> = Lazy::new(|| {
+    let metric = Family::<RpcMethodLabel, Histogram>::new_with_constructor(|| {
+        // Histogram with 10 buckets starting from 0.01s going to 5.12s, each bucket twice as big as the last.
+        Histogram::new(exponential_buckets(0.01, 2.0, 10))
+    });
+    crate::metrics::default_registry().register(
+        "rpc_processing_time",
+        "Duration of RPC method call",
+        metric.clone(),
+    );
+    metric
+});
+
 pub async fn init_prometheus<DB>(
     prometheus_listener: TcpListener,
     db_directory: PathBuf,
@@ -101,6 +128,11 @@ where
     )
 }
 
+#[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
+pub struct RpcMethodLabel {
+    pub method: String,
+}
+
 #[derive(Clone, Debug, Hash, PartialEq, Eq, EncodeLabelSet)]
 pub struct KindLabel {
     kind: &'static str,

src/rpc/metrics_layer.rs

@@ -0,0 +1,57 @@
+// Copyright 2019-2024 ChainSafe Systems
+// SPDX-License-Identifier: Apache-2.0, MIT
+
+use crate::metrics;
+use futures::future::BoxFuture;
+use futures::FutureExt;
+use jsonrpsee::server::middleware::rpc::RpcServiceT;
+use jsonrpsee::MethodResponse;
+use tower::Layer;
+
+// State-less jsonrpcsee layer for measuring RPC metrics
+#[derive(Clone)]
+pub struct MetricsLayer {}
+
+impl<S> Layer<S> for MetricsLayer {
+    type Service = RecordMetrics<S>;
+
+    fn layer(&self, service: S) -> Self::Service {
+        RecordMetrics { service }
+    }
+}
+
+#[derive(Clone)]
+pub struct RecordMetrics<S> {
+    service: S,
+}
+
+impl<'a, S> RpcServiceT<'a> for RecordMetrics<S>
+where
+    S: RpcServiceT<'a> + Send + Sync + Clone + 'static,
+{
+    type Future = BoxFuture<'a, MethodResponse>;
+
+    fn call(&self, req: jsonrpsee::types::Request<'a>) -> Self::Future {
+        let service = self.service.clone();
+        let method = metrics::RpcMethodLabel {
+            method: req.method_name().to_owned(),
+        };
+
+        async move {
+            // Cannot use HistogramTimerExt::start_timer here since it would lock the metric.
+            let start_time = std::time::Instant::now();
+            let req = service.call(req).await;
+
+            metrics::RPC_METHOD_TIME
+                .get_or_create(&method)
+                .observe(start_time.elapsed().as_secs_f64());
+
+            if req.is_error() {
+                metrics::RPC_METHOD_FAILURE.get_or_create(&method).inc();
+            }
+
+            req
+        }
+        .boxed()
+    }
+}

src/rpc/mod.rs

@@ -4,6 +4,7 @@
 mod auth_layer;
 mod channel;
 mod client;
+mod metrics_layer;
 mod request;
 
 pub use client::Client;
@@ -298,6 +299,7 @@ use crate::key_management::KeyStore;
 use crate::rpc::auth_layer::AuthLayer;
 use crate::rpc::channel::RpcModule as FilRpcModule;
 pub use crate::rpc::channel::CANCEL_METHOD_NAME;
+use crate::rpc::metrics_layer::MetricsLayer;
 
 use crate::blocks::Tipset;
 use fvm_ipld_blockstore::Blockstore;
@@ -436,10 +438,12 @@ where
                 // NOTE, the rpc middleware must be initialized here to be able to created once per connection
                 // with data from the connection such as the headers in this example
                 let headers = req.headers().clone();
-                let rpc_middleware = RpcServiceBuilder::new().layer(AuthLayer {
-                    headers,
-                    keystore: keystore.clone(),
-                });
+                let rpc_middleware = RpcServiceBuilder::new()
+                    .layer(AuthLayer {
+                        headers,
+                        keystore: keystore.clone(),
+                    })
+                    .layer(MetricsLayer {});
                 let mut jsonrpsee_svc = svc_builder
                     .set_rpc_middleware(rpc_middleware)
                     .build(methods, stop_handle);