Exports block query errors to prometheus metrics (counter) (#1239)

* separate by type * add help info * remove new line in help and fix readme * feedback
cosmos · Jul 25, 2023 · 107d3f5 · 107d3f5
1 parent 55084bd
commit 107d3f5
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 1 deletion.
diff --git a/docs/advanced_usage.md b/docs/advanced_usage.md
@@ -5,7 +5,7 @@
 **Prometheus exporter**
 
 If you started `rly` with the default `--debug-addr` argument,
-you can use `http://$IP:7597/relayer/metrics` as a target for your prometheus scraper.
+you can use `http://$IP:5183/relayer/metrics` as a target for your prometheus scraper.
 
 **Example metrics**
 

diff --git a/relayer/chains/cosmos/cosmos_chain_processor.go b/relayer/chains/cosmos/cosmos_chain_processor.go
@@ -386,12 +386,18 @@ func (ccp *CosmosChainProcessor) queryCycle(ctx context.Context, persistence *qu
 			queryCtx, cancelQueryCtx := context.WithTimeout(ctx, blockResultsQueryTimeout)
 			defer cancelQueryCtx()
 			blockRes, err = ccp.chainProvider.RPCClient.BlockResults(queryCtx, &i)
+			if err != nil && ccp.metrics != nil {
+				ccp.metrics.IncBlockQueryFailure(chainID, "RPC Client")
+			}
 			return err
 		})
 		eg.Go(func() (err error) {
 			queryCtx, cancelQueryCtx := context.WithTimeout(ctx, queryTimeout)
 			defer cancelQueryCtx()
 			ibcHeader, err = ccp.chainProvider.QueryIBCHeader(queryCtx, i)
+			if err != nil && ccp.metrics != nil {
+				ccp.metrics.IncBlockQueryFailure(chainID, "IBC Header")
+			}
 			return err
 		})
 

diff --git a/relayer/processor/metrics.go b/relayer/processor/metrics.go
@@ -14,6 +14,7 @@ type PrometheusMetrics struct {
 	LatestHeightGauge     *prometheus.GaugeVec
 	WalletBalance         *prometheus.GaugeVec
 	FeesSpent             *prometheus.GaugeVec
+	BlockQueryFailure     *prometheus.CounterVec
 	ClientExpiration      *prometheus.GaugeVec
 }
 
@@ -41,9 +42,14 @@ func (m *PrometheusMetrics) SetClientExpiration(pathName, chain, clientID, trust
 	m.ClientExpiration.WithLabelValues(pathName, chain, clientID, trustingPeriod).Set(timeToExpiration.Seconds())
 }
 
+func (m *PrometheusMetrics) IncBlockQueryFailure(chain, err string) {
+	m.BlockQueryFailure.WithLabelValues(chain, err).Inc()
+}
+
 func NewPrometheusMetrics() *PrometheusMetrics {
 	packetLabels := []string{"path", "chain", "channel", "port", "type"}
 	heightLabels := []string{"chain"}
+	blockQueryFailureLabels := []string{"chain", "type"}
 	walletLabels := []string{"chain", "gas_price", "key", "address", "denom"}
 	clientExpirationLables := []string{"path_name", "chain", "client_id", "trusting_period"}
 	registry := prometheus.NewRegistry()
@@ -70,6 +76,10 @@ func NewPrometheusMetrics() *PrometheusMetrics {
 			Name: "cosmos_relayer_fees_spent",
 			Help: "The amount of fees spent from the relayer's wallet",
 		}, walletLabels),
+		BlockQueryFailure: registerer.NewCounterVec(prometheus.CounterOpts{
+			Name: "cosmos_relayer_block_query_errors_total",
+			Help: "The total number of block query failures. The failures are separated into two catagories: 'RPC Client' and 'IBC Header'",
+		}, blockQueryFailureLabels),
 		ClientExpiration: registerer.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "cosmos_relayer_client_expiration_seconds",
 			Help: "Seconds until the client expires",