Configurable Client Errors (#12760)

* Implement client errors

* Test client errors

* Set client errors

* changeset

* Fix race conditions

* Fix config tests

* Enable client errors

* remove global map

* Move errors to NodePool

* Rename errors

* Update chains-evm.toml

* Fix test data

* Fix and generate docs

* generate testdata

* changeset

* update changeset

* update testdata

* generate docs

* Update errors.go

* remove variable

* Refactor ClientErrors

* Pass client errors

* update client errors

* Update config

* Fix config

* Update config tests

* update golden files

* Update config-multi-chain-effective.toml

* Add regex to toml

* Update lovely-jeans-confess.md

.changeset/lovely-jeans-confess.md

@@ -0,0 +1,6 @@
+---
+"chainlink": minor
+---
+
+#nops : Enable configurable client error regexes for error classification
+#added : New toml configuration options for [EVM.NodePool.Errors] to pass regexes on NonceTooLow, NonceTooHigh, ReplacementTransactionUnderpriced, LimitReached, TransactionAlreadyInMempool, TerminallyUnderpriced, InsufficientEth, TxFeeExceedsCap, L2FeeTooLow, L2FeeTooHigh, L2Full, TransactionAlreadyMined, Fatal, and ServiceUnavailable.

common/client/node.go

@@ -9,6 +9,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/config"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 
@@ -45,6 +47,7 @@ type NodeConfig interface {
 	SyncThreshold() uint32
 	NodeIsSyncingEnabled() bool
 	FinalizedBlockPollInterval() time.Duration
+	Errors() config.ClientErrors
 }
 
 type ChainConfig interface {

common/client/node_test.go

@@ -9,6 +9,7 @@ import (
 
 	clientMocks "github.com/smartcontractkit/chainlink/v2/common/client/mocks"
 	"github.com/smartcontractkit/chainlink/v2/common/types"
+	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/config"
 )
 
 type testNodeConfig struct {
@@ -18,6 +19,7 @@ type testNodeConfig struct {
 	syncThreshold              uint32
 	nodeIsSyncingEnabled       bool
 	finalizedBlockPollInterval time.Duration
+	errors                     config.ClientErrors
 }
 
 func (n testNodeConfig) PollFailureThreshold() uint32 {
@@ -44,6 +46,10 @@ func (n testNodeConfig) FinalizedBlockPollInterval() time.Duration {
 	return n.finalizedBlockPollInterval
 }
 
+func (n testNodeConfig) Errors() config.ClientErrors {
+	return n.errors
+}
+
 type testNode struct {
 	*node[types.ID, Head, NodeClient[types.ID, Head]]
 }

core/chains/evm/client/chain_client.go

@@ -5,6 +5,8 @@ import (
 	"math/big"
 	"time"
 
+	evmconfig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/config"
+
 	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -38,8 +40,9 @@ type chainClient struct {
 		RPCClient,
 		rpc.BatchElem,
 	]
-	logger    logger.SugaredLogger
-	chainType config.ChainType
+	logger       logger.SugaredLogger
+	chainType    config.ChainType
+	clientErrors evmconfig.ClientErrors
 }
 
 func NewChainClient(
@@ -51,6 +54,7 @@ func NewChainClient(
 	sendonlys []commonclient.SendOnlyNode[*big.Int, RPCClient],
 	chainID *big.Int,
 	chainType config.ChainType,
+	clientErrors evmconfig.ClientErrors,
 ) Client {
 	multiNode := commonclient.NewMultiNode(
 		lggr,
@@ -63,13 +67,14 @@ func NewChainClient(
 		chainType,
 		"EVM",
 		func(tx *types.Transaction, err error) commonclient.SendTxReturnCode {
-			return ClassifySendError(err, logger.Sugared(logger.Nop()), tx, common.Address{}, chainType.IsL2())
+			return ClassifySendError(err, clientErrors, logger.Sugared(logger.Nop()), tx, common.Address{}, chainType.IsL2())
 		},
 		0, // use the default value provided by the implementation
 	)
 	return &chainClient{
-		multiNode: multiNode,
-		logger:    logger.Sugared(lggr),
+		multiNode:    multiNode,
+		logger:       logger.Sugared(lggr),
+		clientErrors: clientErrors,
 	}
 }
 
@@ -208,7 +213,7 @@ func (c *chainClient) SendTransaction(ctx context.Context, tx *types.Transaction
 
 func (c *chainClient) SendTransactionReturnCode(ctx context.Context, tx *types.Transaction, fromAddress common.Address) (commonclient.SendTxReturnCode, error) {
 	err := c.SendTransaction(ctx, tx)
-	returnCode := ClassifySendError(err, c.logger, tx, fromAddress, c.IsL2())
+	returnCode := ClassifySendError(err, c.clientErrors, c.logger, tx, fromAddress, c.IsL2())
 	return returnCode, err
 }
 

core/chains/evm/client/client.go

@@ -225,7 +225,7 @@ func (client *client) HeaderByHash(ctx context.Context, h common.Hash) (*types.H
 
 func (client *client) SendTransactionReturnCode(ctx context.Context, tx *types.Transaction, fromAddress common.Address) (commonclient.SendTxReturnCode, error) {
 	err := client.SendTransaction(ctx, tx)
-	returnCode := ClassifySendError(err, client.logger, tx, fromAddress, client.pool.ChainType().IsL2())
+	returnCode := ClassifySendError(err, nil, client.logger, tx, fromAddress, client.pool.ChainType().IsL2())
 	return returnCode, err
 }
 

core/chains/evm/client/config_builder_test.go

@@ -56,7 +56,7 @@ func TestClientConfigBuilder(t *testing.T) {
 	require.Equal(t, *finalityTagEnabled, chainCfg.FinalityTagEnabled())
 
 	// let combiler tell us, when we do not have sufficient data to create evm client
-	_ = client.NewEvmClient(nodePool, chainCfg, logger.Test(t), big.NewInt(10), nodes)
+	_ = client.NewEvmClient(nodePool, chainCfg, nil, logger.Test(t), big.NewInt(10), nodes)
 }
 
 func TestNodeConfigs(t *testing.T) {