Skip to content

Commit

Permalink
Export Client Trusting Period to Prometheus metrics (#1246)
Browse files Browse the repository at this point in the history 
* export client trusting period

* update docs
  • Loading branch information
@boojamya
boojamya committed Jul 27, 2023
1 parent cdd7661 commit 5236f6f
Show file tree
Hide file tree
Showing 3 changed files with 25 additions and 28 deletions.
40 changes: 13 additions & 27 deletions docs/advanced_usage.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,33 +10,19 @@ you can use `http://$IP:5183/relayer/metrics` as a target for your prometheus sc

Exported metrics:

| **Exported Metric** | **Description** | **Type** |
|:----------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------:|
| cosmos_relayer_observed_packets | The total number of observed packets | Counter |
| cosmos_relayer_relayed_packets | The total number of relayed packets | Counter |
| cosmos_relayer_chain_latest_height | The current height of the chain | Gauge |
| cosmos_relayer_wallet_balance | The current balance for the relayer's wallet | Gauge |
| cosmos_relayer_fees_spent | The amount of fees spent from the relayer's wallet | Gauge |
| cosmos_relayer_tx_failure | <br>The total number of tx failures broken up into catagories .<br>Categories:<br> - "packet messages are redundant"<br> - "insufficient funds"<br> - "invalid coins"<br> - "out of gas"<br> - "incorrect account sequence" <br><br> "Tx Failure" is the the catch all bucket| Counter |
**Example metrics**

```
go_goroutines 29
...
go_threads 39
...
observed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="acknowledge_packet"} 57
observed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="recv_packet"} 103
observed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="send_packet"} 58
observed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="acknowledge_packet"} 107
observed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="recv_packet"} 60
observed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="send_packet"} 102
...
relayed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="acknowledge_packet"} 31
relayed_packets{chain="cosmoshub-4",channel="channel-141",path="hubosmo",port="transfer",type="recv_packet"} 65
relayed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="acknowledge_packet"} 36
relayed_packets{chain="osmosis-1",channel="channel-0",path="hubosmo",port="transfer",type="recv_packet"} 35
```
| **Exported Metric** | **Description** | **Type** |
|:---------------------------------------------: |:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: |:--------: |
| cosmos_relayer_observed_packets | The total number of observed packets | Counter |
| cosmos_relayer_relayed_packets | The total number of relayed packets | Counter |
| cosmos_relayer_chain_latest_height | The current height of the chain | Gauge |
| cosmos_relayer_wallet_balance | The current balance for the relayer's wallet | Gauge |
| cosmos_relayer_fees_spent | The amount of fees spent from the relayer's wallet | Gauge |
| cosmos_relayer_tx_failure | <br>The total number of tx failures broken up into categories:<br> - "packet messages are redundant"<br> - "insufficient funds"<br> - "invalid coins"<br> - "out of gas"<br><br><br>"Tx Failure" is the the catch all bucket | Counter |
| cosmos_relayer_block_query_errors_total | The total number of block query failures. The failures are separated into two categories:<br> - "RPC Client"<br> - "IBC Header" | Counter |
| cosmos_relayer_client_expiration_seconds | Seconds until the client expires | Gauge |
| cosmos_relayer_client_trusting_period_seconds | The trusting period (in seconds) of the client | Gauge |



---

Expand Down
1 change: 1 addition & 0 deletions relayer/processor/message_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ func (mp *messageProcessor) shouldUpdateClientNow(ctx context.Context, src, dst
if mp.metrics != nil {
timeToExpiration := dst.clientState.TrustingPeriod - time.Since(consensusHeightTime)
mp.metrics.SetClientExpiration(src.info.PathName, dst.info.ChainID, dst.clientState.ClientID, fmt.Sprint(dst.clientState.TrustingPeriod.String()), timeToExpiration)
mp.metrics.SetClientTrustingPeriod(src.info.PathName, dst.info.ChainID, dst.info.ClientID, time.Duration(dst.clientState.TrustingPeriod))
}

if shouldUpdateClientNow {
Expand Down
12 changes: 11 additions & 1 deletion relayer/processor/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ type PrometheusMetrics struct {
TxFailureError *prometheus.CounterVec
BlockQueryFailure *prometheus.CounterVec
ClientExpiration *prometheus.GaugeVec
ClientTrustingPeriod *prometheus.GaugeVec
}

func (m *PrometheusMetrics) AddPacketsObserved(path, chain, channel, port, eventType string, count int) {
Expand All @@ -43,6 +44,10 @@ func (m *PrometheusMetrics) SetClientExpiration(pathName, chain, clientID, trust
m.ClientExpiration.WithLabelValues(pathName, chain, clientID, trustingPeriod).Set(timeToExpiration.Seconds())
}

func (m *PrometheusMetrics) SetClientTrustingPeriod(pathName, chain, clientID string, trustingPeriod time.Duration) {
m.ClientTrustingPeriod.WithLabelValues(pathName, chain, clientID).Set(trustingPeriod.Abs().Seconds())
}

func (m *PrometheusMetrics) IncBlockQueryFailure(chain, err string) {
m.BlockQueryFailure.WithLabelValues(chain, err).Inc()
}
Expand All @@ -58,6 +63,7 @@ func NewPrometheusMetrics() *PrometheusMetrics {
blockQueryFailureLabels := []string{"chain", "type"}
walletLabels := []string{"chain", "gas_price", "key", "address", "denom"}
clientExpirationLables := []string{"path_name", "chain", "client_id", "trusting_period"}
clientTrustingPeriodLables := []string{"path_name", "chain", "client_id"}
registry := prometheus.NewRegistry()
registerer := promauto.With(registry)
return &PrometheusMetrics{
Expand Down Expand Up @@ -88,11 +94,15 @@ func NewPrometheusMetrics() *PrometheusMetrics {
}, txFailureLabels),
BlockQueryFailure: registerer.NewCounterVec(prometheus.CounterOpts{
Name: "cosmos_relayer_block_query_errors_total",
Help: "The total number of block query failures. The failures are separated into two catagories: 'RPC Client' and 'IBC Header'",
Help: "The total number of block query failures. The failures are separated into two categories: 'RPC Client' and 'IBC Header'",
}, blockQueryFailureLabels),
ClientExpiration: registerer.NewGaugeVec(prometheus.GaugeOpts{
Name: "cosmos_relayer_client_expiration_seconds",
Help: "Seconds until the client expires",
}, clientExpirationLables),
ClientTrustingPeriod: registerer.NewGaugeVec(prometheus.GaugeOpts{
Name: "cosmos_relayer_client_trusting_period_seconds",
Help: "The trusting period (in seconds) of the client",
}, clientTrustingPeriodLables),
}
}

0 comments on commit 5236f6f

Please sign in to comment.