fix(rpc): workaround for tonic hanging on large error message (#18639)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

src/common/metrics/Cargo.toml

@@ -17,6 +17,7 @@ normal = ["workspace-hack"]
 [dependencies]
 auto_impl = "1"
 bytes = "1"
+cfg-or-panic = "0.2"
 clap = { workspace = true }
 easy-ext = "1"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }

src/common/metrics/src/monitor/connection.rs

@@ -22,6 +22,7 @@ use std::sync::LazyLock;
 use std::task::{Context, Poll};
 use std::time::Duration;
 
+use cfg_or_panic::cfg_or_panic;
 use futures::FutureExt;
 use http::Uri;
 use hyper_util::client::legacy::connect::dns::{GaiAddrs, GaiFuture, GaiResolver, Name};
@@ -591,23 +592,40 @@ impl Service<Name> for MonitoredGaiResolver {
     }
 }
 
+#[cfg_or_panic(not(madsim))]
+fn monitored_http_connector(
+    connection_type: impl Into<String>,
+    config: TcpConfig,
+) -> MonitoredConnection<HttpConnector<MonitoredGaiResolver>, MonitorNewConnectionImpl> {
+    let resolver = MonitoredGaiResolver::default();
+    let mut http = HttpConnector::new_with_resolver(resolver);
+
+    http.enforce_http(false);
+    http.set_nodelay(config.tcp_nodelay);
+    http.set_keepalive(config.keepalive_duration);
+
+    monitor_connector(http, connection_type)
+}
+
+/// Attach general configurations to the endpoint.
+#[cfg_or_panic(not(madsim))]
+fn configure_endpoint(endpoint: Endpoint) -> Endpoint {
+    // This is to mitigate https://github.com/risingwavelabs/risingwave/issues/18039.
+    // TODO: remove this after https://github.com/hyperium/hyper/issues/3724 gets resolved.
+    endpoint.http2_max_header_list_size(16 * 1024 * 1024)
+}
+
 #[easy_ext::ext(EndpointExt)]
 impl Endpoint {
     pub async fn monitored_connect(
-        self,
+        mut self,
         connection_type: impl Into<String>,
         config: TcpConfig,
     ) -> Result<Channel, tonic::transport::Error> {
         #[cfg(not(madsim))]
         {
-            let resolver = MonitoredGaiResolver::default();
-            let mut http = HttpConnector::new_with_resolver(resolver);
-
-            http.enforce_http(false);
-            http.set_nodelay(config.tcp_nodelay);
-            http.set_keepalive(config.keepalive_duration);
-
-            let connector = monitor_connector(http, connection_type);
+            self = configure_endpoint(self);
+            let connector = monitored_http_connector(connection_type, config);
             self.connect_with_connector(connector).await
         }
         #[cfg(madsim)]
@@ -618,16 +636,12 @@ impl Endpoint {
 
     #[cfg(not(madsim))]
     pub fn monitored_connect_lazy(
-        self,
+        mut self,
         connection_type: impl Into<String>,
         config: TcpConfig,
     ) -> Channel {
-        let mut http = HttpConnector::new();
-        http.enforce_http(false);
-        http.set_nodelay(config.tcp_nodelay);
-        http.set_keepalive(config.keepalive_duration);
-
-        let connector = monitor_connector(http, connection_type);
+        self = configure_endpoint(self);
+        let connector = monitored_http_connector(connection_type, config);
         self.connect_with_connector_lazy(connector)
     }
 }