EVM-850 Validator rootchain balance metrics (#1941)

---------

Co-authored-by: Stefan Negovanović <stefan@ethernal.tech>

command/server/config/config.go

@@ -39,6 +39,8 @@ type Config struct {
 
 	ConcurrentRequestsDebug uint64 `json:"concurrent_requests_debug" yaml:"concurrent_requests_debug"`
 	WebSocketReadLimit      uint64 `json:"web_socket_read_limit" yaml:"web_socket_read_limit"`
+
+	MetricsInterval time.Duration `json:"metrics_interval" yaml:"metrics_interval"`
 }
 
 // Telemetry holds the config details for metric services.
@@ -96,6 +98,10 @@ const (
 	// DefaultRelayerTrackerPollInterval specifies time interval after which relayer node's event tracker
 	// polls child chain to get the latest block
 	DefaultRelayerTrackerPollInterval time.Duration = time.Second
+
+	// DefaultMetricsInterval specifies the time interval after which Prometheus metrics will be generated.
+	// A value of 0 means the metrics are disabled.
+	DefaultMetricsInterval time.Duration = time.Second * 8
 )
 
 // DefaultConfig returns the default server configuration
@@ -136,6 +142,7 @@ func DefaultConfig() *Config {
 		ConcurrentRequestsDebug:    DefaultConcurrentRequestsDebug,
 		WebSocketReadLimit:         DefaultWebSocketReadLimit,
 		RelayerTrackerPollInterval: DefaultRelayerTrackerPollInterval,
+		MetricsInterval:            DefaultMetricsInterval,
 	}
 }
 

command/server/params.go

@@ -45,6 +45,8 @@ const (
 	webSocketReadLimitFlag      = "websocket-read-limit"
 
 	relayerTrackerPollIntervalFlag = "relayer-poll-interval"
+
+	metricsIntervalFlag = "metrics-interval"
 )
 
 // Flags that are deprecated, but need to be preserved for
@@ -190,5 +192,6 @@ func (p *serverParams) generateConfig() *server.Config {
 		Relayer:                    p.relayer,
 		NumBlockConfirmations:      p.rawConfig.NumBlockConfirmations,
 		RelayerTrackerPollInterval: p.rawConfig.RelayerTrackerPollInterval,
+		MetricsInterval:            p.rawConfig.MetricsInterval,
 	}
 }

command/server/server.go

@@ -249,6 +249,13 @@ func setFlags(cmd *cobra.Command) {
 		"interval (number of seconds) at which relayer's tracker polls for latest block at childchain",
 	)
 
+	cmd.Flags().DurationVar(
+		&params.rawConfig.MetricsInterval,
+		metricsIntervalFlag,
+		defaultConfig.MetricsInterval,
+		"the interval (in seconds) at which special metrics are generated. a value of zero means the metrics are disabled",
+	)
+
 	setLegacyFlags(cmd)
 
 	setDevFlags(cmd)

consensus/consensus.go

@@ -3,6 +3,7 @@ package consensus
 import (
 	"context"
 	"log"
+	"time"
 
 	"github.com/0xPolygon/polygon-edge/blockchain"
 	"github.com/0xPolygon/polygon-edge/chain"
@@ -78,6 +79,7 @@ type Params struct {
 	BlockTime      uint64
 
 	NumBlockConfirmations uint64
+	MetricsInterval       time.Duration
 }
 
 // Factory is the factory function to create a discovery consensus

consensus/polybft/checkpoint_manager.go

@@ -15,7 +15,6 @@ import (
 	"github.com/0xPolygon/polygon-edge/merkle-tree"
 	"github.com/0xPolygon/polygon-edge/txrelayer"
 	"github.com/0xPolygon/polygon-edge/types"
-	metrics "github.com/armon/go-metrics"
 	hclog "github.com/hashicorp/go-hclog"
 	"github.com/umbracle/ethgo"
 )
@@ -98,33 +97,31 @@ func newCheckpointManager(key ethgo.Key, checkpointOffset uint64,
 	}
 }
 
-// getLatestCheckpointBlock queries CheckpointManager smart contract and retrieves latest checkpoint block number
-func (c *checkpointManager) getLatestCheckpointBlock() (uint64, error) {
-	checkpointBlockNumMethodEncoded, err := currentCheckpointBlockNumMethod.Encode([]interface{}{})
+// getCurrentCheckpointBlock queries CheckpointManager smart contract and retrieves the current checkpoint block number
+func getCurrentCheckpointBlock(relayer txrelayer.TxRelayer, checkpointManagerAddr types.Address) (uint64, error) {
+	checkpointBlockNumInput, err := currentCheckpointBlockNumMethod.Encode([]interface{}{})
 	if err != nil {
-		return 0, fmt.Errorf("failed to encode currentCheckpointId function parameters: %w", err)
+		return 0, fmt.Errorf("failed to encode currentCheckpointBlockNumber function parameters: %w", err)
 	}
 
-	latestCheckpointBlockRaw, err := c.rootChainRelayer.Call(
-		c.key.Address(),
-		ethgo.Address(c.checkpointManagerAddr),
-		checkpointBlockNumMethodEncoded)
+	currentCheckpointBlockRaw, err := relayer.Call(ethgo.ZeroAddress, ethgo.Address(checkpointManagerAddr),
+		checkpointBlockNumInput)
 	if err != nil {
-		return 0, fmt.Errorf("failed to invoke currentCheckpointId function on the rootchain: %w", err)
+		return 0, fmt.Errorf("failed to invoke currentCheckpointBlockNumber function on the rootchain: %w", err)
 	}
 
-	latestCheckpointBlockNum, err := strconv.ParseUint(latestCheckpointBlockRaw, 0, 64)
+	currentCheckpointBlock, err := strconv.ParseUint(currentCheckpointBlockRaw, 0, 64)
 	if err != nil {
-		return 0, fmt.Errorf("failed to convert current checkpoint id '%s' to number: %w",
-			latestCheckpointBlockRaw, err)
+		return 0, fmt.Errorf("failed to convert current checkpoint block number '%s' to number: %w",
+			currentCheckpointBlockRaw, err)
 	}
 
-	return latestCheckpointBlockNum, nil
+	return currentCheckpointBlock, nil
 }
 
 // submitCheckpoint sends a transaction with checkpoint data to the rootchain
 func (c *checkpointManager) submitCheckpoint(latestHeader *types.Header, isEndOfEpoch bool) error {
-	lastCheckpointBlockNumber, err := c.getLatestCheckpointBlock()
+	lastCheckpointBlockNumber, err := getCurrentCheckpointBlock(c.rootChainRelayer, c.checkpointManagerAddr)
 	if err != nil {
 		return err
 	}
@@ -234,8 +231,6 @@ func (c *checkpointManager) encodeAndSendCheckpoint(header *types.Header, extra
 		return fmt.Errorf("checkpoint submission transaction failed for block %d", header.Number)
 	}
 
-	// update checkpoint block number metrics
-	metrics.SetGauge([]string{"bridge", "checkpoint_block_number"}, float32(header.Number))
 	c.logger.Debug("send checkpoint txn success", "block number", header.Number, "gasUsed", receipt.GasUsed)
 
 	return nil

consensus/polybft/checkpoint_manager_test.go

@@ -195,13 +195,13 @@ func TestCheckpointManager_getCurrentCheckpointID(t *testing.T) {
 			name:         "Rootchain call returns an error",
 			checkpointID: "",
 			returnError:  errors.New("internal error"),
-			errSubstring: "failed to invoke currentCheckpointId function on the rootchain",
+			errSubstring: "failed to invoke currentCheckpointBlockNumber function on the rootchain",
 		},
 		{
 			name:         "Failed to parse return value from rootchain",
 			checkpointID: "Hello World!",
 			returnError:  error(nil),
-			errSubstring: "failed to convert current checkpoint id",
+			errSubstring: "failed to convert current checkpoint block number",
 		},
 	}
 
@@ -222,7 +222,8 @@ func TestCheckpointManager_getCurrentCheckpointID(t *testing.T) {
 				key:              acc.Ecdsa,
 				logger:           hclog.NewNullLogger(),
 			}
-			actualCheckpointID, err := checkpointMgr.getLatestCheckpointBlock()
+			actualCheckpointID, err := getCurrentCheckpointBlock(checkpointMgr.rootChainRelayer,
+				checkpointMgr.checkpointManagerAddr)
 			if c.errSubstring == "" {
 				expectedCheckpointID, err := strconv.ParseUint(c.checkpointID, 0, 64)
 				require.NoError(t, err)

consensus/polybft/polybft.go

@@ -10,6 +10,8 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/hashicorp/go-hclog"
+
 	"github.com/0xPolygon/polygon-edge/chain"
 	"github.com/0xPolygon/polygon-edge/consensus"
 	"github.com/0xPolygon/polygon-edge/consensus/polybft/contractsapi"
@@ -24,7 +26,6 @@ import (
 	"github.com/0xPolygon/polygon-edge/state"
 	"github.com/0xPolygon/polygon-edge/syncer"
 	"github.com/0xPolygon/polygon-edge/types"
-	"github.com/hashicorp/go-hclog"
 )
 
 const (
@@ -156,7 +157,7 @@ func GenesisPostHookFactory(config *chain.Chain, engineName string) func(txn *st
 			proxyAddrMapping[contracts.RewardTokenContract] = contracts.RewardTokenContractV1
 		}
 
-		if err = setUpProxies(transition, polyBFTConfig.ProxyContractsAdmin, proxyAddrMapping); err != nil {
+		if err = initProxies(transition, polyBFTConfig.ProxyContractsAdmin, proxyAddrMapping); err != nil {
 			return err
 		}
 
@@ -527,6 +528,9 @@ func (p *Polybft) Start() error {
 	// start state DB process
 	go p.state.startStatsReleasing()
 
+	// polybft rootchain metrics
+	go p.publishRootchainMetrics(p.logger.Named("rootchain_metrics"))
+
 	return nil
 }
 
@@ -781,7 +785,8 @@ func (p *Polybft) FilterExtra(extra []byte) ([]byte, error) {
 	return GetIbftExtraClean(extra)
 }
 
-func setUpProxies(transition *state.Transition, admin types.Address,
+// initProxies initializes proxy contracts, that allow upgradeability of contracts implementation
+func initProxies(transition *state.Transition, admin types.Address,
 	proxyToImplMap map[types.Address]types.Address) error {
 	for proxyAddress, implAddress := range proxyToImplMap {
 		protectSetupProxyFn := &contractsapi.ProtectSetUpProxyGenesisProxyFn{Initiator: contracts.SystemCaller}

consensus/polybft/state_stats.go → consensus/polybft/stats.go

@@ -1,10 +1,14 @@
 package polybft
 
 import (
+	"math/big"
 	"time"
 
+	"github.com/0xPolygon/polygon-edge/txrelayer"
 	"github.com/armon/go-metrics"
+	"github.com/hashicorp/go-hclog"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/umbracle/ethgo"
 )
 
 // startStatsReleasing starts the process that releases BoltDB stats into prometheus periodically.
@@ -154,3 +158,53 @@ func (s *State) startStatsReleasing() {
 		prev = stats
 	}
 }
+
+// publishRootchainMetrics publishes rootchain related metrics
+func (p *Polybft) publishRootchainMetrics(logger hclog.Logger) {
+	interval := p.config.MetricsInterval
+	validatorAddr := p.key.Address()
+	bridgeCfg := p.consensusConfig.Bridge
+
+	// zero means metrics are disabled
+	if interval <= 0 {
+		return
+	}
+
+	relayer, err := txrelayer.NewTxRelayer(txrelayer.WithIPAddress(bridgeCfg.JSONRPCEndpoint))
+	if err != nil {
+		logger.Error("failed to connect to the rootchain node", "err", err, "JSON RPC", bridgeCfg.JSONRPCEndpoint)
+
+		return
+	}
+
+	gweiPerWei := new(big.Float).SetInt(new(big.Int).Exp(big.NewInt(10), big.NewInt(9), nil)) // 10^9
+
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-p.closeCh:
+			return
+		case <-ticker.C:
+			// rootchain validator balance
+			balance, err := relayer.Client().Eth().GetBalance(p.key.Address(), ethgo.Latest)
+			if err != nil {
+				logger.Error("failed to query eth_getBalance", "err", err)
+			} else {
+				balanceInGwei := new(big.Float).Quo(new(big.Float).SetInt(balance), gweiPerWei)
+				balanceInGweiFloat, _ := balanceInGwei.Float32()
+
+				metrics.SetGauge([]string{"bridge", "validator_root_balance_gwei", validatorAddr.String()}, balanceInGweiFloat)
+			}
+
+			// rootchain current checkpoint block
+			checkpointBlock, err := getCurrentCheckpointBlock(relayer, bridgeCfg.CheckpointManagerAddr)
+			if err != nil {
+				logger.Error("failed to query latest checkpoint block", "err", err)
+			} else {
+				metrics.SetGauge([]string{"bridge", "checkpoint_block_number"}, float32(checkpointBlock))
+			}
+		}
+	}
+}

server/config.go

@@ -46,6 +46,7 @@ type Config struct {
 
 	NumBlockConfirmations      uint64
 	RelayerTrackerPollInterval time.Duration
+	MetricsInterval            time.Duration
 }
 
 // Telemetry holds the config details for metric services

server/server.go

@@ -585,6 +585,7 @@ func (s *Server) setupConsensus() error {
 			SecretsManager:        s.secretsManager,
 			BlockTime:             uint64(blockTime.Seconds()),
 			NumBlockConfirmations: s.config.NumBlockConfirmations,
+			MetricsInterval:       s.config.MetricsInterval,
 		},
 	)