feat: add metrics for failed deliveries

crates/engine/tree/src/tree/metrics.rs

@@ -34,6 +34,14 @@ pub(crate) struct EngineMetrics {
     pub(crate) new_payload_messages: Counter,
     /// Histogram of persistence operation durations (in seconds)
     pub(crate) persistence_duration: Histogram,
+    /// Tracks the how often we failed to deliver a newPayload response.
+    ///
+    /// This effectively tracks how often the message sender dropped the channel and indicates a CL
+    /// request timeout (e.g. it took more than 8s to send the response and the CL terminated the
+    /// request which resulted in a closed channel).
+    pub(crate) failed_new_payload_response_deliveries: Counter,
+    /// Tracks the how often we failed to deliver a forkchoice update response.
+    pub(crate) failed_forkchoice_updated_response_deliveries: Counter,
     // TODO add latency metrics
 }
 

crates/engine/tree/src/tree/mod.rs

@@ -1219,6 +1219,10 @@ where
                                 if let Err(err) =
                                     tx.send(output.map(|o| o.outcome).map_err(Into::into))
                                 {
+                                    self.metrics
+                                        .engine
+                                        .failed_forkchoice_updated_response_deliveries
+                                        .increment(1);
                                     error!(target: "engine::tree", "Failed to send event: {err:?}");
                                 }
                             }
@@ -1230,6 +1234,10 @@ where
                                     )
                                 })) {
                                     error!(target: "engine::tree", "Failed to send event: {err:?}");
+                                    self.metrics
+                                        .engine
+                                        .failed_new_payload_response_deliveries
+                                        .increment(1);
                                 }
                             }
                             BeaconEngineMessage::TransitionConfigurationExchanged => {