feat: cosmos redundant rpcs

cmd/appstate.go

@@ -319,3 +319,15 @@ func (a *appState) useRpcAddr(chainName string, rpcAddr string) error {
 		return nil
 	})
 }
+
+func (a *appState) useBackupRpcAddrs(chainName string, rpcAddrs []string) error {
+	_, exists := a.config.Chains[chainName]
+	if !exists {
+		return fmt.Errorf("chain %s not found in config", chainName)
+	}
+
+	return a.performConfigLockingOperation(context.Background(), func() error {
+		a.config.Chains[chainName].ChainProvider.SetBackupRpcAddrs(rpcAddrs)
+		return nil
+	})
+}

cmd/chains.go

@@ -41,16 +41,17 @@ func chainsCmd(a *appState) *cobra.Command {
 		chainsAddDirCmd(a),
 		cmdChainsConfigure(a),
 		cmdChainsUseRpcAddr(a),
+		cmdChainsUseBackupRpcAddr(a),
 	)
 
 	return cmd
 }
 
 func cmdChainsUseRpcAddr(a *appState) *cobra.Command {
 	cmd := &cobra.Command{
-		Use:     "set-rpc-addr chain_name  valid_rpc_url",
+		Use:     "set-rpc-addr chain_name valid_rpc_url",
 		Aliases: []string{"rpc"},
-		Short:   "Sets  chain's rpc address",
+		Short:   "Sets chain's rpc address",
 		Args:    withUsage(cobra.ExactArgs(2)),
 		Example: strings.TrimSpace(fmt.Sprintf(`
 $ %s chains set-rpc-addr ibc-0 https://abc.xyz.com:443
@@ -69,6 +70,36 @@ $ %s ch set-rpc-addr ibc-0 https://abc.xyz.com:443`, appName, appName)),
 	return cmd
 }
 
+func cmdChainsUseBackupRpcAddr(a *appState) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:     "set-backup-rpc-addr chain_name comma_separated_valid_rpc_urls",
+		Aliases: []string{"rpc"},
+		Short:   "Sets chain's backup rpc addresses",
+		Args:    withUsage(cobra.ExactArgs(2)),
+		Example: strings.TrimSpace(fmt.Sprintf(`
+$ %s chains set-backup-rpc-addr ibc-0 https://abc.xyz.com:443,https://123.456.com:443
+$ %s ch set-backup-rpc-addr ibc-0 https://abc.xyz.com:443,https://123.456.com:443`, appName, appName)),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			chainName := args[0]
+			rpc_addresses := args[1]
+
+			// Split rpc_addresses by ','
+			rpc_addresses_list := strings.Split(rpc_addresses, ",")
+
+			// Loop through and ensure valid
+			for _, rpc_address := range rpc_addresses_list {
+				if !isValidURL(rpc_address) {
+					return invalidRpcAddr(rpc_address)
+				}
+			}
+
+			return a.useBackupRpcAddrs(chainName, rpc_addresses_list)
+		},
+	}
+
+	return cmd
+}
+
 func chainsAddrCmd(a *appState) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "address chain_name",

relayer/chain.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/avast/retry-go/v4"
-	"net/url"
 	"time"
 
+	"github.com/avast/retry-go/v4"
+
 	"github.com/cosmos/cosmos-sdk/crypto/hd"
 	clienttypes "github.com/cosmos/ibc-go/v8/modules/core/02-client/types"
 	"github.com/cosmos/relayer/v2/relayer/provider"
@@ -31,7 +31,6 @@ type Chain struct {
 
 	ChainProvider provider.ChainProvider
 	Chainid       string `yaml:"chain-id" json:"chain-id"`
-	RPCAddr       string `yaml:"rpc-addr" json:"rpc-addr"`
 
 	PathEnd *PathEnd `yaml:"-" json:"-"`
 
@@ -137,12 +136,6 @@ func (c Chains) Gets(chainIDs ...string) (map[string]*Chain, error) {
 	return out, nil
 }
 
-// GetRPCPort returns the port configured for the chain
-func (c *Chain) GetRPCPort() string {
-	u, _ := url.Parse(c.RPCAddr)
-	return u.Port()
-}
-
 // CreateTestKey creates a key for test chain
 func (c *Chain) CreateTestKey() error {
 	if c.ChainProvider.KeyExists(c.ChainProvider.Key()) {

relayer/chains/cosmos/provider.go

@@ -38,6 +38,7 @@ type CosmosProviderConfig struct {
 	ChainName        string                     `json:"-" yaml:"-"`
 	ChainID          string                     `json:"chain-id" yaml:"chain-id"`
 	RPCAddr          string                     `json:"rpc-addr" yaml:"rpc-addr"`
+	BackupRPCAddrs   []string                   `json:"backup-rpc-addrs" yaml:"backup-rpc-addrs"`
 	AccountPrefix    string                     `json:"account-prefix" yaml:"account-prefix"`
 	KeyringBackend   string                     `json:"keyring-backend" yaml:"keyring-backend"`
 	DynamicGasPrice  bool                       `json:"dynamic-gas-price" yaml:"dynamic-gas-price"`
@@ -273,6 +274,13 @@ func (cc *CosmosProvider) SetRpcAddr(rpcAddr string) error {
 	return nil
 }
 
+// SetBackupRpcAddrs sets the backup rpc-addr for the chain.
+// These addrs are used in the event that the primary rpc-addr is down.
+func (cc *CosmosProvider) SetBackupRpcAddrs(rpcAddrs []string) error {
+	cc.PCfg.BackupRPCAddrs = rpcAddrs
+	return nil
+}
+
 // Init initializes the keystore, RPC client, amd light client provider.
 // Once initialization is complete an attempt to query the underlying node's tendermint version is performed.
 // NOTE: Init must be called after creating a new instance of CosmosProvider.
@@ -295,22 +303,123 @@ func (cc *CosmosProvider) Init(ctx context.Context) error {
 		return err
 	}
 
-	c, err := client.NewClient(cc.PCfg.RPCAddr, timeout)
+	// set the RPC client
+	err = cc.setRpcClient(true, cc.PCfg.RPCAddr, timeout)
 	if err != nil {
 		return err
 	}
 
-	lightprovider, err := prov.New(cc.PCfg.ChainID, cc.PCfg.RPCAddr)
+	// set the light client provider
+	err = cc.setLightProvider(cc.PCfg.RPCAddr)
 	if err != nil {
 		return err
 	}
 
-	rpcClient := cwrapper.NewRPCClient(c)
-
-	cc.RPCClient = rpcClient
-	cc.LightProvider = lightprovider
+	// set keybase
 	cc.Keybase = keybase
 
+	// go routine to monitor RPC liveliness
+	go cc.startLivelinessChecks(ctx, timeout)
+
+	return nil
+}
+
+// startLivelinessChecks frequently checks the liveliness of an RPC client and resorts to backup rpcs if the active rpc is down.
+// This is a blocking function; call this within a go routine.
+func (cc *CosmosProvider) startLivelinessChecks(ctx context.Context, timeout time.Duration) {
+	// list of rpcs & index to keep track of active rpc
+	rpcs := append([]string{cc.PCfg.RPCAddr}, cc.PCfg.BackupRPCAddrs...)
+	index := 0
+
+	// exit routine if there is only one rpc client
+	if len(rpcs) <= 1 {
+		cc.log.Debug("No backup RPCs defined", zap.String("chain", cc.ChainName()))
+		return
+	}
+
+	// log the number of available rpcs
+	cc.log.Debug("Available RPC clients", zap.String("chain", cc.ChainName()), zap.Int("count", len(rpcs)))
+
+	// tick every 10s to ensure rpc liveliness
+	ticker := time.NewTicker(10 * time.Second)
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			_, err := cc.RPCClient.Status(ctx)
+			if err != nil {
+				cc.log.Error("RPC client disconnected", zap.String("chain", cc.ChainName()), zap.Error(err))
+
+				attempts := 0
+
+				// attempt to connect to the backup RPC client
+				for {
+
+					attempts++
+					if attempts > len(rpcs) {
+						cc.log.Error("All configured RPC endpoints return non-200 response", zap.String("chain", cc.ChainName()), zap.Error(err))
+						break
+					}
+
+					// get next rpc
+					index = (index + 1) % len(rpcs)
+					rpcAddr := rpcs[index]
+
+					cc.log.Info("Attempting to connect to new RPC", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr))
+
+					// attempt to setup rpc client
+					if err = cc.setRpcClient(false, rpcAddr, timeout); err != nil {
+						cc.log.Debug("Failed to connect to RPC client", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err))
+						continue
+					}
+
+					// attempt to setup light client
+					if err = cc.setLightProvider(rpcAddr); err != nil {
+						cc.log.Debug("Failed to connect to light client provider", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr), zap.Error(err))
+						continue
+					}
+
+					cc.log.Info("Successfully connected to new RPC", zap.String("chain", cc.ChainName()), zap.String("rpc", rpcAddr))
+
+					// rpc found, escape
+					break
+				}
+			}
+		}
+	}
+}
+
+// setRpcClient sets the RPC client for the chain.
+func (cc *CosmosProvider) setRpcClient(onStartup bool, rpcAddr string, timeout time.Duration) error {
+	c, err := client.NewClient(rpcAddr, timeout)
+	if err != nil {
+		return err
+	}
+
+	cc.RPCClient = cwrapper.NewRPCClient(c)
+
+	// Only check status if not on startup, to ensure the relayer will not block on startup.
+	// All subsequent calls will perform the status check to ensure RPC endpoints are rotated
+	// as necessary.
+	if !onStartup {
+		if _, err = cc.RPCClient.Status(context.Background()); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// setLightProvider sets the light client provider for the chain.
+func (cc *CosmosProvider) setLightProvider(rpcAddr string) error {
+	lightprovider, err := prov.New(cc.PCfg.ChainID, rpcAddr)
+	if err != nil {
+		return err
+	}
+
+	cc.LightProvider = lightprovider
 	return nil
 }
 

relayer/chains/penumbra/provider.go

@@ -45,6 +45,7 @@ type PenumbraProviderConfig struct {
 	ChainName        string                     `json:"-" yaml:"-"`
 	ChainID          string                     `json:"chain-id" yaml:"chain-id"`
 	RPCAddr          string                     `json:"rpc-addr" yaml:"rpc-addr"`
+	BackupRPCAddrs   []string                   `json:"backup-rpc-addrs" yaml:"backup-rpc-addrs"`
 	AccountPrefix    string                     `json:"account-prefix" yaml:"account-prefix"`
 	KeyringBackend   string                     `json:"keyring-backend" yaml:"keyring-backend"`
 	GasAdjustment    float64                    `json:"gas-adjustment" yaml:"gas-adjustment"`
@@ -234,6 +235,12 @@ func (cc *PenumbraProvider) SetRpcAddr(rpcAddr string) error {
 	return nil
 }
 
+// SetBackupRpcAddrs implements provider.ChainProvider.
+func (cc *PenumbraProvider) SetBackupRpcAddrs(rpcAddrs []string) error {
+	cc.PCfg.BackupRPCAddrs = rpcAddrs
+	return nil
+}
+
 // Init initializes the keystore, RPC client, amd light client provider.
 // Once initialization is complete an attempt to query the underlying node's tendermint version is performed.
 // NOTE: Init must be called after creating a new instance of CosmosProvider.

relayer/provider/provider.go

@@ -409,6 +409,7 @@ type ChainProvider interface {
 	Sprint(toPrint proto.Message) (string, error)
 
 	SetRpcAddr(rpcAddr string) error
+	SetBackupRpcAddrs(rpcAddrs []string) error
 }
 
 // Do we need intermediate types? i.e. can we use the SDK types for both substrate and cosmos?