diff --git a/docs/advanced_usage.md b/docs/advanced_usage.md index 715bc283d..99a721634 100644 --- a/docs/advanced_usage.md +++ b/docs/advanced_usage.md @@ -5,7 +5,7 @@ **Prometheus exporter** If you started `rly` with the default `--debug-addr` argument, -you can use `http://$IP:7597/relayer/metrics` as a target for your prometheus scraper. +you can use `http://$IP:5183/relayer/metrics` as a target for your prometheus scraper. **Example metrics** diff --git a/relayer/chains/cosmos/cosmos_chain_processor.go b/relayer/chains/cosmos/cosmos_chain_processor.go index 9189e945d..c582cc731 100644 --- a/relayer/chains/cosmos/cosmos_chain_processor.go +++ b/relayer/chains/cosmos/cosmos_chain_processor.go @@ -386,12 +386,18 @@ func (ccp *CosmosChainProcessor) queryCycle(ctx context.Context, persistence *qu queryCtx, cancelQueryCtx := context.WithTimeout(ctx, blockResultsQueryTimeout) defer cancelQueryCtx() blockRes, err = ccp.chainProvider.RPCClient.BlockResults(queryCtx, &i) + if err != nil && ccp.metrics != nil { + ccp.metrics.IncBlockQueryFailure(chainID, "RPC Client") + } return err }) eg.Go(func() (err error) { queryCtx, cancelQueryCtx := context.WithTimeout(ctx, queryTimeout) defer cancelQueryCtx() ibcHeader, err = ccp.chainProvider.QueryIBCHeader(queryCtx, i) + if err != nil && ccp.metrics != nil { + ccp.metrics.IncBlockQueryFailure(chainID, "IBC Header") + } return err }) diff --git a/relayer/processor/metrics.go b/relayer/processor/metrics.go index 6c77b5f3e..27e9e789f 100644 --- a/relayer/processor/metrics.go +++ b/relayer/processor/metrics.go @@ -14,6 +14,7 @@ type PrometheusMetrics struct { LatestHeightGauge *prometheus.GaugeVec WalletBalance *prometheus.GaugeVec FeesSpent *prometheus.GaugeVec + BlockQueryFailure *prometheus.CounterVec ClientExpiration *prometheus.GaugeVec } @@ -41,9 +42,14 @@ func (m *PrometheusMetrics) SetClientExpiration(pathName, chain, clientID, trust m.ClientExpiration.WithLabelValues(pathName, chain, clientID, trustingPeriod).Set(timeToExpiration.Seconds()) } +func (m *PrometheusMetrics) IncBlockQueryFailure(chain, err string) { + m.BlockQueryFailure.WithLabelValues(chain, err).Inc() +} + func NewPrometheusMetrics() *PrometheusMetrics { packetLabels := []string{"path", "chain", "channel", "port", "type"} heightLabels := []string{"chain"} + blockQueryFailureLabels := []string{"chain", "type"} walletLabels := []string{"chain", "gas_price", "key", "address", "denom"} clientExpirationLables := []string{"path_name", "chain", "client_id", "trusting_period"} registry := prometheus.NewRegistry() @@ -70,6 +76,10 @@ func NewPrometheusMetrics() *PrometheusMetrics { Name: "cosmos_relayer_fees_spent", Help: "The amount of fees spent from the relayer's wallet", }, walletLabels), + BlockQueryFailure: registerer.NewCounterVec(prometheus.CounterOpts{ + Name: "cosmos_relayer_block_query_errors_total", + Help: "The total number of block query failures. The failures are separated into two catagories: 'RPC Client' and 'IBC Header'", + }, blockQueryFailureLabels), ClientExpiration: registerer.NewGaugeVec(prometheus.GaugeOpts{ Name: "cosmos_relayer_client_expiration_seconds", Help: "Seconds until the client expires",