Count in-flight GRPC requests (#4489)

Looking at PR #4488, it seems very suspicious that we need to raise GRPC limit. Counting how many in-flight requests we have per request path can allow to shed some light and potentially show the problem.

Cardinality of created counters is very low - O(num of grpc methods).

Even thought we are potentially going to migrate to new networking layer, I think having those counters will help medium term.

crates/sui-node/src/lib.rs

@@ -5,6 +5,7 @@ use anyhow::anyhow;
 use anyhow::bail;
 use anyhow::Result;
 use futures::TryFutureExt;
+use mysten_network::server::ServerBuilder;
 use parking_lot::Mutex;
 use prometheus::Registry;
 use std::option::Option::None;
@@ -39,6 +40,7 @@ use sui_types::messages::{CertifiedTransaction, CertifiedTransactionEffects};
 use tokio::sync::mpsc::channel;
 use tracing::{error, info};
 
+use crate::metrics::GrpcMetrics;
 use sui_core::authority_client::NetworkAuthorityClientMetrics;
 use sui_core::epoch::epoch_store::EpochStore;
 use sui_json_rpc::event_api::EventReadApiImpl;
@@ -278,7 +280,8 @@ impl SuiNode {
             let mut server_conf = mysten_network::config::Config::new();
             server_conf.global_concurrency_limit = config.grpc_concurrency_limit;
             server_conf.load_shed = config.grpc_load_shed;
-            let mut server_builder = server_conf.server_builder();
+            let mut server_builder =
+                ServerBuilder::from_config(&server_conf, GrpcMetrics::new(&prometheus_registry));
 
             if let Some(validator_service) = validator_service {
                 server_builder =

crates/sui-node/src/metrics.rs

@@ -2,8 +2,11 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use axum::{extract::Extension, http::StatusCode, routing::get, Router};
-use prometheus::{Registry, TextEncoder};
+use mysten_network::metrics::MetricsCallbackProvider;
+use prometheus::{register_int_gauge_vec_with_registry, IntGaugeVec, Registry, TextEncoder};
 use std::net::SocketAddr;
+use std::time::Duration;
+use sui_network::tonic::Code;
 
 const METRICS_ROUTE: &str = "/metrics";
 
@@ -34,3 +37,32 @@ async fn metrics(Extension(registry): Extension<Registry>) -> (StatusCode, Strin
         ),
     }
 }
+
+#[derive(Clone)]
+pub struct GrpcMetrics {
+    inflight_grpc: IntGaugeVec,
+}
+
+impl GrpcMetrics {
+    pub fn new(registry: &Registry) -> Self {
+        Self {
+            inflight_grpc: register_int_gauge_vec_with_registry!(
+                "inflight_grpc",
+                "Total in-flight GRPC per route",
+                &["path"],
+                registry,
+            )
+            .unwrap(),
+        }
+    }
+}
+
+impl MetricsCallbackProvider for GrpcMetrics {
+    fn on_request(&self, path: String) {
+        self.inflight_grpc.with_label_values(&[&path]).inc();
+    }
+
+    fn on_response(&self, path: String, _latency: Duration, _status: u16, _grpc_status_code: Code) {
+        self.inflight_grpc.with_label_values(&[&path]).dec();
+    }
+}