Skip to content

Commit

Permalink
Implement transaction sender tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@DylanTinianov
DylanTinianov committed Jun 25, 2024
1 parent 17f77c0 commit e5a619e
Show file tree
Hide file tree
Showing 5 changed files with 614 additions and 321 deletions.
5 changes: 4 additions & 1 deletion common/client/multi_node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ func (c *MultiNode[CHAIN_ID, RPC_CLIENT]) ChainID() CHAIN_ID {
}

func (c *MultiNode[CHAIN_ID, RPC_CLIENT]) DoAll(ctx context.Context, do func(ctx context.Context, rpc RPC_CLIENT, isSendOnly bool) bool) error {
ctx, cancel := c.chStop.CtxCancel(context.WithCancel(ctx))
defer cancel()

callsCompleted := 0
for _, n := range c.primaryNodes {
if ctx.Err() != nil {
Expand All @@ -118,7 +121,7 @@ func (c *MultiNode[CHAIN_ID, RPC_CLIENT]) DoAll(ctx context.Context, do func(ctx
if n.State() != NodeStateAlive {
continue
}
do(ctx, n.RPC(), false)
do(ctx, n.RPC(), true)
}
return nil
}
Expand Down
303 changes: 0 additions & 303 deletions common/client/multi_node_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,306 +480,3 @@ func TestMultiNode_nLiveNodes(t *testing.T) {
})
}
}

/* TODO: Move test coverate to TransactionSender
func TestMultiNode_SendTransaction(t *testing.T) {
t.Parallel()
classifySendTxError := func(tx any, err error) SendTxReturnCode {
if err != nil {
return Fatal
}
return Successful
}
newNodeWithState := func(t *testing.T, state NodeState, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, types.Head[Hashable], multiNodeRPCClient] {
rpc := newMultiNodeRPCClient(t)
rpc.On("SendTransaction", mock.Anything, mock.Anything).Return(txErr).Run(sendTxRun).Maybe()
node := newMockNode[types.ID, types.Head[Hashable], multiNodeRPCClient](t)
node.On("String").Return("node name").Maybe()
node.On("RPC").Return(rpc).Maybe()
node.On("State").Return(state).Maybe()
node.On("Close").Return(nil).Once()
return node
}
newNode := func(t *testing.T, txErr error, sendTxRun func(args mock.Arguments)) *mockNode[types.ID, types.Head[Hashable], multiNodeRPCClient] {
return newNodeWithState(t, NodeStateAlive, txErr, sendTxRun)
}
newStartedMultiNode := func(t *testing.T, opts multiNodeOpts) testMultiNode {
mn := newTestMultiNode(t, opts)
err := mn.StartOnce("startedTestMultiNode", func() error { return nil })
require.NoError(t, err)
t.Cleanup(func() {
require.NoError(t, mn.Close())
})
return mn
}
t.Run("Fails if there is no nodes available", func(t *testing.T) {
mn := newStartedMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: types.RandomID(),
})
err := mn.SendTransaction(tests.Context(t), nil)
assert.EqualError(t, err, ErroringNodeError.Error())
})
t.Run("Transaction failure happy path", func(t *testing.T) {
chainID := types.RandomID()
expectedError := errors.New("transaction failed")
mainNode := newNode(t, expectedError, nil)
lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel)
mn := newStartedMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: chainID,
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{mainNode},
sendonlys: []SendOnlyNode[types.ID, multiNodeRPCClient]{newNode(t, errors.New("unexpected error"), nil)},
classifySendTxError: classifySendTxError,
logger: lggr,
})
err := mn.SendTransaction(tests.Context(t), nil)
require.EqualError(t, err, expectedError.Error())
tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2)
tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 2)
})
t.Run("Transaction success happy path", func(t *testing.T) {
chainID := types.RandomID()
mainNode := newNode(t, nil, nil)
lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel)
mn := newStartedMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: chainID,
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{mainNode},
sendonlys: []SendOnlyNode[types.ID, multiNodeRPCClient]{newNode(t, errors.New("unexpected error"), nil)},
classifySendTxError: classifySendTxError,
logger: lggr,
})
err := mn.SendTransaction(tests.Context(t), nil)
require.NoError(t, err)
tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2)
tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 1)
})
t.Run("Context expired before collecting sufficient results", func(t *testing.T) {
chainID := types.RandomID()
testContext, testCancel := context.WithCancel(tests.Context(t))
defer testCancel()
mainNode := newNode(t, errors.New("unexpected error"), func(_ mock.Arguments) {
// block caller til end of the test
<-testContext.Done()
})
mn := newStartedMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: chainID,
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{mainNode},
classifySendTxError: classifySendTxError,
})
requestContext, cancel := context.WithCancel(tests.Context(t))
cancel()
err := mn.SendTransaction(requestContext, nil)
require.EqualError(t, err, "context canceled")
})
t.Run("Soft timeout stops results collection", func(t *testing.T) {
chainID := types.RandomID()
expectedError := errors.New("tmp error")
fastNode := newNode(t, expectedError, nil)
// hold reply from the node till end of the test
testContext, testCancel := context.WithCancel(tests.Context(t))
defer testCancel()
slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) {
// block caller til end of the test
<-testContext.Done()
})
mn := newStartedMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: chainID,
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{fastNode, slowNode},
classifySendTxError: classifySendTxError,
sendTxSoftTimeout: tests.TestInterval,
})
err := mn.SendTransaction(tests.Context(t), nil)
require.EqualError(t, err, expectedError.Error())
})
t.Run("Returns success without waiting for the rest of the nodes", func(t *testing.T) {
chainID := types.RandomID()
fastNode := newNode(t, nil, nil)
// hold reply from the node till end of the test
testContext, testCancel := context.WithCancel(tests.Context(t))
defer testCancel()
slowNode := newNode(t, errors.New("transaction failed"), func(_ mock.Arguments) {
// block caller til end of the test
<-testContext.Done()
})
slowSendOnly := newNode(t, errors.New("send only failed"), func(_ mock.Arguments) {
// block caller til end of the test
<-testContext.Done()
})
lggr, observedLogs := logger.TestObserved(t, zap.WarnLevel)
mn := newTestMultiNode(t, multiNodeOpts{
logger: lggr,
selectionMode: NodeSelectionModeRoundRobin,
chainID: chainID,
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{fastNode, slowNode},
sendonlys: []SendOnlyNode[types.ID, multiNodeRPCClient]{slowSendOnly},
classifySendTxError: classifySendTxError,
sendTxSoftTimeout: tests.TestInterval,
})
assert.NoError(t, mn.StartOnce("startedTestMultiNode", func() error { return nil }))
err := mn.SendTransaction(tests.Context(t), nil)
require.NoError(t, err)
testCancel()
require.NoError(t, mn.Close())
tests.AssertLogEventually(t, observedLogs, "observed invariant violation on SendTransaction")
})
t.Run("Fails when closed", func(t *testing.T) {
mn := newTestMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: types.RandomID(),
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{newNode(t, nil, nil)},
sendonlys: []SendOnlyNode[types.ID, multiNodeRPCClient]{newNode(t, nil, nil)},
classifySendTxError: classifySendTxError,
})
err := mn.StartOnce("startedTestMultiNode", func() error { return nil })
require.NoError(t, err)
require.NoError(t, mn.Close())
err = mn.SendTransaction(tests.Context(t), nil)
require.EqualError(t, err, "aborted while broadcasting tx - MultiNode is stopped: context canceled")
})
t.Run("Returns error if there is no healthy primary nodes", func(t *testing.T) {
mn := newStartedMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: types.RandomID(),
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{newNodeWithState(t, NodeStateUnreachable, nil, nil)},
sendonlys: []SendOnlyNode[types.ID, multiNodeRPCClient]{newNodeWithState(t, NodeStateUnreachable, nil, nil)},
classifySendTxError: classifySendTxError,
})
err := mn.SendTransaction(tests.Context(t), nil)
assert.EqualError(t, err, ErroringNodeError.Error())
})
t.Run("Transaction success even if one of the nodes is unhealthy", func(t *testing.T) {
chainID := types.RandomID()
mainNode := newNode(t, nil, nil)
unexpectedCall := func(args mock.Arguments) {
panic("SendTx must not be called for unhealthy node")
}
unhealthyNode := newNodeWithState(t, NodeStateUnreachable, nil, unexpectedCall)
unhealthySendOnlyNode := newNodeWithState(t, NodeStateUnreachable, nil, unexpectedCall)
lggr, observedLogs := logger.TestObserved(t, zap.DebugLevel)
mn := newStartedMultiNode(t, multiNodeOpts{
selectionMode: NodeSelectionModeRoundRobin,
chainID: chainID,
nodes: []Node[types.ID, types.Head[Hashable], multiNodeRPCClient]{mainNode, unhealthyNode},
sendonlys: []SendOnlyNode[types.ID, multiNodeRPCClient]{unhealthySendOnlyNode, newNode(t, errors.New("unexpected error"), nil)},
classifySendTxError: classifySendTxError,
logger: lggr,
})
err := mn.SendTransaction(tests.Context(t), nil)
require.NoError(t, err)
tests.AssertLogCountEventually(t, observedLogs, "Node sent transaction", 2)
tests.AssertLogCountEventually(t, observedLogs, "RPC returned error", 1)
})
}
func TestMultiNode_SendTransaction_aggregateTxResults(t *testing.T) {
t.Parallel()
// ensure failure on new SendTxReturnCode
codesToCover := map[SendTxReturnCode]struct{}{}
for code := Successful; code < sendTxReturnCodeLen; code++ {
codesToCover[code] = struct{}{}
}
testCases := []struct {
Name string
ExpectedTxResult string
ExpectedCriticalErr string
ResultsByCode sendTxErrors
}{
{
Name: "Returns success and logs critical error on success and Fatal",
ExpectedTxResult: "success",
ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error",
ResultsByCode: sendTxErrors{
Successful: {errors.New("success")},
Fatal: {errors.New("fatal")},
},
},
{
Name: "Returns TransactionAlreadyKnown and logs critical error on TransactionAlreadyKnown and Fatal",
ExpectedTxResult: "tx_already_known",
ExpectedCriticalErr: "found contradictions in nodes replies on SendTransaction: got success and severe error",
ResultsByCode: sendTxErrors{
TransactionAlreadyKnown: {errors.New("tx_already_known")},
Unsupported: {errors.New("unsupported")},
},
},
{
Name: "Prefers sever error to temporary",
ExpectedTxResult: "underpriced",
ExpectedCriticalErr: "",
ResultsByCode: sendTxErrors{
Retryable: {errors.New("retryable")},
Underpriced: {errors.New("underpriced")},
},
},
{
Name: "Returns temporary error",
ExpectedTxResult: "retryable",
ExpectedCriticalErr: "",
ResultsByCode: sendTxErrors{
Retryable: {errors.New("retryable")},
},
},
{
Name: "Insufficient funds is treated as error",
ExpectedTxResult: "",
ExpectedCriticalErr: "",
ResultsByCode: sendTxErrors{
Successful: {nil},
InsufficientFunds: {errors.New("insufficientFunds")},
},
},
{
Name: "Logs critical error on empty ResultsByCode",
ExpectedTxResult: "expected at least one response on SendTransaction",
ExpectedCriticalErr: "expected at least one response on SendTransaction",
ResultsByCode: sendTxErrors{},
},
{
Name: "Zk out of counter error",
ExpectedTxResult: "not enough keccak counters to continue the execution",
ExpectedCriticalErr: "",
ResultsByCode: sendTxErrors{
OutOfCounters: {errors.New("not enough keccak counters to continue the execution")},
},
},
}
for _, testCase := range testCases {
for code := range testCase.ResultsByCode {
delete(codesToCover, code)
}
t.Run(testCase.Name, func(t *testing.T) {
txResult, err := aggregateTxResults(testCase.ResultsByCode)
if testCase.ExpectedTxResult == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, txResult, testCase.ExpectedTxResult)
}
logger.Sugared(logger.Test(t)).Info("Map: " + fmt.Sprint(testCase.ResultsByCode))
logger.Sugared(logger.Test(t)).Criticalw("observed invariant violation on SendTransaction", "resultsByCode", testCase.ResultsByCode, "err", err)
if testCase.ExpectedCriticalErr == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, testCase.ExpectedCriticalErr)
}
})
}
// explicitly signal that following codes are properly handled in aggregateTxResults,
//but dedicated test cases won't be beneficial
for _, codeToIgnore := range []SendTxReturnCode{Unknown, ExceedsMaxFee, FeeOutOfValidRange} {
delete(codesToCover, codeToIgnore)
}
assert.Empty(t, codesToCover, "all of the SendTxReturnCode must be covered by this test")
}
*/
Loading

0 comments on commit e5a619e

Please sign in to comment.