Added more metrics (#868)

config.toml

@@ -38,20 +38,6 @@ fee_amount = 1000
 clock_drift = '5s'
 trusting_period = '14days'
 
-[[chains]]
-id = 'cosmoshub-4'
-rpc_addr = 'http://cosmos.decentrox.com:26657'
-grpc_addr = 'http://cosmos.decentrox.com:9090'
-websocket_addr = 'ws://cosmos.decentrox.com:26657/websocket'
-account_prefix = 'cosmos'
-key_name = 'testkey'
-store_prefix = 'ibc'
-gas = 3000000
-fee_denom = 'uatom'
-fee_amount = 300
-clock_drift = '5s'
-trusting_period = '14days'
-
 [chains.trust_threshold]
 numerator = '1'
 denominator = '3'

relayer/src/supervisor.rs

@@ -110,6 +110,8 @@ impl Supervisor {
                 }
                 IbcEvent::WriteAcknowledgement(ref packet) => {
                     if let Ok(object) = Object::for_write_ack(packet, src_chain) {
+                        // TODO: Find a better place to record the telemetry metric
+                        let _ = self.telemetry.send(MetricUpdate::AcknowledgePacket(1));
                         collected.per_object.entry(object).or_default().push(event);
                     }
                 }
@@ -137,19 +139,23 @@ impl Supervisor {
             .map(|c| c.id.clone())
             .collect_vec();
 
-        let _ = self.telemetry.send(MetricUpdate::RelayChainsNumber(chain_ids.len() as u64));
-
         for chain_id in chain_ids {
             let chain = match self.registry.get_or_spawn(&chain_id) {
-                Ok(chain_handle) => chain_handle,
+                Ok(chain_handle) => {
+                    let _ = self.telemetry.send(MetricUpdate::RelayChainsNumber(1));
+                    chain_handle
+                },
                 Err(e) => {
                     error!("skipping workers for chain id {}. reason: failed to spawn chain runtime with error: {}", chain_id, e);
                     continue;
                 }
             };
 
             let channels = match chain.query_channels(req.clone()) {
-                Ok(channels) => channels,
+                Ok(channels) => {
+                    let _ = self.telemetry.send(MetricUpdate::RelayChannelsNumber(1));
+                    channels
+                },
                 Err(e) => {
                     error!("failed to query channels from {}: {}", chain_id, e);
                     continue;

relayer/src/telemetry/server.rs

@@ -1,5 +1,4 @@
 use prometheus::{Encoder, TextEncoder};
-use tracing::info;
 use crate::telemetry::state::TelemetryState;
 
 pub struct TelemetryServer {
@@ -16,13 +15,10 @@ impl TelemetryServer {
             router!(request,
                 // The prometheus endpoint
                 (GET) (/metrics) => {
-                    //telemetry_state.packets_relayed.add(1);
-                    info!("metrics called on telemetry server");
                     let mut buffer = vec![];
                     let encoder = TextEncoder::new();
                     let metric_families = telemetry_state.exporter.registry().gather();
                     encoder.encode(&metric_families, &mut buffer).unwrap();
-                    dbg!(metric_families);
                     rouille::Response::from_data(encoder.format_type().to_string(), buffer)
                 },
 

relayer/src/telemetry/service.rs

@@ -2,7 +2,10 @@ use crossbeam_channel::Receiver;
 use crate::telemetry::state::TelemetryState;
 
 pub enum MetricUpdate {
-    RelayChainsNumber(u64)
+    RelayChainsNumber(u64),
+    RelayChannelsNumber(u64),
+    AcknowledgePacket(u64),
+    TxCount(u64),
 }
 
 pub struct TelemetryService {
@@ -27,6 +30,9 @@ impl TelemetryService {
     fn apply_update(&self, update: MetricUpdate) {
         match update {
             MetricUpdate::RelayChainsNumber(n) => self.state.relay_chains_num.add(n ),
+            MetricUpdate::RelayChannelsNumber(n) => self.state.relay_channels_num.add(n),
+            MetricUpdate::AcknowledgePacket(n) => self.state.tx_msg_ibc_acknowledge_packet.add(n),
+            MetricUpdate::TxCount(n) => self.state.tx_count.add(n),
         }
     }
 }

relayer/src/telemetry/state.rs

@@ -11,8 +11,18 @@ lazy_static! {
 pub struct TelemetryState {
     pub exporter: PrometheusExporter,
 
-    // Count the number of trans
+    // Number of chains the relay is connecting to
     pub relay_chains_num: BoundCounter<'static, u64>,
+
+    // Number of channels the relay is connecting to
+    pub relay_channels_num: BoundCounter<'static, u64>,
+
+    // Total number of IBC packets acknowledged
+    pub tx_msg_ibc_acknowledge_packet: BoundCounter<'static, u64>,
+
+    // Total number of txs processed via Relay tx
+    pub tx_count: BoundCounter<'static, u64>,
+
 }
 
 impl TelemetryState {
@@ -22,10 +32,25 @@ impl TelemetryState {
         let telemetry_state = TelemetryState {
             exporter,
             relay_chains_num: meter
-                .u64_counter("relay_chains_num")
+                .u64_counter("hermes_chains_num")
                 .with_description("Number of chains the relay is connecting to")
                 .init()
                 .bind(HANDLER_ALL.as_ref()),
+            relay_channels_num: meter
+                .u64_counter("hermes_channels_num")
+                .with_description("Number of channels the relay is connecting to")
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            tx_msg_ibc_acknowledge_packet: meter
+                .u64_counter("hermes_tx_msg_ibc_acknowledge_packet")
+                .with_description("Total number of IBC packets acknowledged")
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            tx_count: meter
+                .u64_counter("tx_count")
+                .with_description("Total number of txs processed via Relay tx")
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
         };
         telemetry_state
     }