Added more metrics to service. Hookup the packet timeout metric (#868)

relayer/src/supervisor.rs

@@ -100,18 +100,19 @@ impl Supervisor {
                 IbcEvent::SendPacket(ref packet) => {
                     if let Ok(object) = Object::for_send_packet(packet, src_chain) {
                         collected.per_object.entry(object).or_default().push(event);
-                        // Increase counter
                     }
                 }
                 IbcEvent::TimeoutPacket(ref packet) => {
                     if let Ok(object) = Object::for_timeout_packet(packet, src_chain) {
+                        // TODO: Is this the right place to record the telemetry metric ?
+                        let _ = self.telemetry.send(MetricUpdate::TimeoutPacket(1));
                         collected.per_object.entry(object).or_default().push(event);
                     }
                 }
                 IbcEvent::WriteAcknowledgement(ref packet) => {
                     if let Ok(object) = Object::for_write_ack(packet, src_chain) {
-                        // TODO: Find a better place to record the telemetry metric
-                        let _ = self.telemetry.send(MetricUpdate::AcknowledgePacket(1));
+                        // TODO: Is this the right place to record the telemetry metric ?
+                        let _ = self.telemetry.send(MetricUpdate::IbcAcknowledgePacket(1));
                         collected.per_object.entry(object).or_default().push(event);
                     }
                 }

relayer/src/telemetry.rs

@@ -13,7 +13,9 @@ pub fn spawn(port: u16, enabled: bool) -> Sender<MetricUpdate> {
 
     // Only start the telemetry service and server if it is enabled in the configuration
     if enabled {
-        let telemetry_state = TelemetryState::new();
+        let telemetry_state = TelemetryState {
+            ..Default::default()
+        };
         let service = TelemetryService::new(telemetry_state.clone(), rx);
         let server = TelemetryServer::new(telemetry_state.clone());
 

relayer/src/telemetry/server.rs

@@ -10,7 +10,8 @@ impl TelemetryServer {
         TelemetryServer { state }
     }
 
-    pub(crate) fn run(&self, telemetry_state: TelemetryState, listen_port: u16) -> () {
+    #[allow(clippy::manual_strip)]
+    pub(crate) fn run(&self, telemetry_state: TelemetryState, listen_port: u16) {
         rouille::start_server(format!("localhost:{}", listen_port), move |request| {
             router!(request,
                 // The prometheus endpoint

relayer/src/telemetry/service.rs

@@ -4,8 +4,14 @@ use crossbeam_channel::Receiver;
 pub enum MetricUpdate {
     RelayChainsNumber(u64),
     RelayChannelsNumber(u64),
-    AcknowledgePacket(u64),
     TxCount(u64),
+    TxSuccess(u64),
+    TxFailed(u64),
+    IbcAcknowledgePacket(u64),
+    IbcRecvPacket(u64),
+    IbcTransferSend(u64),
+    IbcTransferReceive(u64),
+    TimeoutPacket(u64),
 }
 
 pub struct TelemetryService {
@@ -28,8 +34,16 @@ impl TelemetryService {
         match update {
             MetricUpdate::RelayChainsNumber(n) => self.state.relay_chains_num.add(n),
             MetricUpdate::RelayChannelsNumber(n) => self.state.relay_channels_num.add(n),
-            MetricUpdate::AcknowledgePacket(n) => self.state.tx_msg_ibc_acknowledge_packet.add(n),
+            MetricUpdate::IbcAcknowledgePacket(n) => {
+                self.state.tx_msg_ibc_acknowledge_packet.add(n)
+            }
+            MetricUpdate::IbcRecvPacket(n) => self.state.tx_msg_ibc_recv_packet.add(n),
             MetricUpdate::TxCount(n) => self.state.tx_count.add(n),
+            MetricUpdate::TxSuccess(n) => self.state.tx_successful.add(n),
+            MetricUpdate::TxFailed(n) => self.state.tx_failed.add(n),
+            MetricUpdate::IbcTransferSend(n) => self.state.ibc_transfer_send.add(n),
+            MetricUpdate::IbcTransferReceive(n) => self.state.ibc_transfer_receive.add(n),
+            MetricUpdate::TimeoutPacket(n) => self.state.ibc_timeout_packet.add(n),
         }
     }
 }

relayer/src/telemetry/state.rs

@@ -20,12 +20,30 @@ pub struct TelemetryState {
     // Total number of IBC packets acknowledged
     pub tx_msg_ibc_acknowledge_packet: BoundCounter<'static, u64>,
 
-    // Total number of txs processed via Relay tx
+    // Total number of txs processed via relay tx
     pub tx_count: BoundCounter<'static, u64>,
+
+    // Total number of successful txs processed via relay tx
+    pub tx_successful: BoundCounter<'static, u64>,
+
+    // Total number of failed txs processed via relay tx
+    pub tx_failed: BoundCounter<'static, u64>,
+
+    // Total number of IBC transfers sent from a chain (source or sink)
+    pub ibc_transfer_send: BoundCounter<'static, u64>,
+
+    // Total number of IBC transfers received to a chain (source or sink)
+    pub ibc_transfer_receive: BoundCounter<'static, u64>,
+
+    // Total number of IBC packets received
+    pub tx_msg_ibc_recv_packet: BoundCounter<'static, u64>,
+
+    // Total number of IBC timeout packets
+    pub ibc_timeout_packet: BoundCounter<'static, u64>,
 }
 
-impl TelemetryState {
-    pub fn new() -> TelemetryState {
+impl Default for TelemetryState {
+    fn default() -> Self {
         let exporter = opentelemetry_prometheus::exporter().init();
         let meter = global::meter("hermes");
         let telemetry_state = TelemetryState {
@@ -46,8 +64,42 @@ impl TelemetryState {
                 .init()
                 .bind(HANDLER_ALL.as_ref()),
             tx_count: meter
-                .u64_counter("tx_count")
-                .with_description("Total number of txs processed via Relay tx")
+                .u64_counter("hermes_tx_count")
+                .with_description("Total number of txs processed via relay tx")
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            tx_successful: meter
+                .u64_counter("hermes_tx_successful")
+                .with_description("Total number of successful txs processed via relay tx")
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            tx_failed: meter
+                .u64_counter("hermes_tx_failed")
+                .with_description("Total number of failed txs processed via relay tx")
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            ibc_transfer_send: meter
+                .u64_counter("hermes_ibc_transfer_send")
+                .with_description(
+                    "Total number of IBC transfers sent from a chain (source or sink)",
+                )
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            ibc_transfer_receive: meter
+                .u64_counter("hermes_ibc_transfer_receive")
+                .with_description(
+                    "Total number of IBC transfers received to a chain (source or sink)",
+                )
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            tx_msg_ibc_recv_packet: meter
+                .u64_counter("hermes_tx_msg_ibc_recv_packet")
+                .with_description("Total number of IBC packets received")
+                .init()
+                .bind(HANDLER_ALL.as_ref()),
+            ibc_timeout_packet: meter
+                .u64_counter("hermes_ibc_timeout_packet")
+                .with_description("Total number of IBC timeout packets")
                 .init()
                 .bind(HANDLER_ALL.as_ref()),
         };