diff --git a/api/config.go b/api/config.go index 29036547b4..5ac7ba3b2d 100644 --- a/api/config.go +++ b/api/config.go @@ -21,15 +21,18 @@ type Config struct { GasStation gasstation.Config `yaml:"gasStation"` RangeQueryLimit uint64 `yaml:"rangeQueryLimit"` Tracer tracer.Config `yaml:"tracer"` + // BatchRequestLimit is the maximum number of requests in a batch. + BatchRequestLimit int `yaml:"batchRequestLimit"` } // DefaultConfig is the default config var DefaultConfig = Config{ - UseRDS: false, - GRPCPort: 14014, - HTTPPort: 15014, - WebSocketPort: 16014, - TpsWindow: 10, - GasStation: gasstation.DefaultConfig, - RangeQueryLimit: 1000, + UseRDS: false, + GRPCPort: 14014, + HTTPPort: 15014, + WebSocketPort: 16014, + TpsWindow: 10, + GasStation: gasstation.DefaultConfig, + RangeQueryLimit: 1000, + BatchRequestLimit: _defaultBatchRequestLimit, } diff --git a/api/serverV2.go b/api/serverV2.go index 65b9832826..65b5000de4 100644 --- a/api/serverV2.go +++ b/api/serverV2.go @@ -49,7 +49,7 @@ func NewServerV2( if err != nil { return nil, err } - web3Handler := NewWeb3Handler(coreAPI, cfg.RedisCacheURL) + web3Handler := NewWeb3Handler(coreAPI, cfg.RedisCacheURL, cfg.BatchRequestLimit) tp, err := tracer.NewProvider( tracer.WithServiceName(cfg.Tracer.ServiceName), diff --git a/api/serverV2_test.go b/api/serverV2_test.go index 0727ce1cd9..acb80c3817 100644 --- a/api/serverV2_test.go +++ b/api/serverV2_test.go @@ -19,7 +19,7 @@ func TestServerV2(t *testing.T) { defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) // TODO: mock web3handler - web3Handler := NewWeb3Handler(core, "") + web3Handler := NewWeb3Handler(core, "", _defaultBatchRequestLimit) svr := &ServerV2{ core: core, grpcServer: NewGRPCServer(core, testutil.RandomPort()), diff --git a/api/web3server.go b/api/web3server.go index b65bc07be4..65ca4643fd 100644 --- a/api/web3server.go +++ b/api/web3server.go @@ -33,6 +33,8 @@ import ( const ( _metamaskBalanceContractAddr = "io1k8uw2hrlvnfq8s2qpwwc24ws2ru54heenx8chr" + // _defaultBatchRequestLimit is the default maximum number of items in a batch. + _defaultBatchRequestLimit = 100 // Maximum number of items in a batch. ) type ( @@ -42,8 +44,9 @@ type ( } web3Handler struct { - coreService CoreService - cache apiCache + coreService CoreService + cache apiCache + batchRequestLimit int } ) @@ -71,6 +74,7 @@ var ( errInvalidFilterID = errors.New("filter not found") errInvalidBlock = errors.New("invalid block") errUnsupportedAction = errors.New("the type of action is not supported") + errMsgBatchTooLarge = errors.New("batch too large") _pendingBlockNumber = "pending" _latestBlockNumber = "latest" @@ -82,10 +86,11 @@ func init() { } // NewWeb3Handler creates a handle to process web3 requests -func NewWeb3Handler(core CoreService, cacheURL string) Web3Handler { +func NewWeb3Handler(core CoreService, cacheURL string, batchRequestLimit int) Web3Handler { return &web3Handler{ - coreService: core, - cache: newAPICache(15*time.Minute, cacheURL), + coreService: core, + cache: newAPICache(15*time.Minute, cacheURL), + batchRequestLimit: batchRequestLimit, } } @@ -102,8 +107,18 @@ func (svr *web3Handler) HandlePOSTReq(ctx context.Context, reader io.Reader, wri if !web3Reqs.IsArray() { return svr.handleWeb3Req(ctx, &web3Reqs, writer) } - batchWriter := apitypes.NewBatchWriter(writer) web3ReqArr := web3Reqs.Array() + if len(web3ReqArr) > int(svr.batchRequestLimit) { + err := errors.Wrapf( + errMsgBatchTooLarge, + "batch size %d exceeds the limit %d", + len(web3ReqArr), + svr.batchRequestLimit, + ) + span.RecordError(err) + return writer.Write(&web3Response{err: err}) + } + batchWriter := apitypes.NewBatchWriter(writer) for i := range web3ReqArr { if err := svr.handleWeb3Req(ctx, &web3ReqArr[i], batchWriter); err != nil { return err diff --git a/api/web3server_integrity_test.go b/api/web3server_integrity_test.go index b15d0c58de..ab146924ed 100644 --- a/api/web3server_integrity_test.go +++ b/api/web3server_integrity_test.go @@ -42,7 +42,7 @@ func TestWeb3ServerIntegrity(t *testing.T) { ctx := context.Background() web3svr.Start(ctx) defer web3svr.Stop(ctx) - handler := newHTTPHandler(NewWeb3Handler(svr.core, "")) + handler := newHTTPHandler(NewWeb3Handler(svr.core, "", _defaultBatchRequestLimit)) // send request t.Run("eth_gasPrice", func(t *testing.T) { diff --git a/api/web3server_test.go b/api/web3server_test.go index 929ecf9ef7..4bfdccdee2 100644 --- a/api/web3server_test.go +++ b/api/web3server_test.go @@ -89,7 +89,7 @@ func TestHandlePost(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - svr := newHTTPHandler(NewWeb3Handler(core, "")) + svr := newHTTPHandler(NewWeb3Handler(core, "", _defaultBatchRequestLimit)) getServerResp := func(svr *hTTPHandler, req *http.Request) *httptest.ResponseRecorder { req.Header.Set("Content-Type", "application/json") resp := httptest.NewRecorder() @@ -123,8 +123,8 @@ func TestHandlePost(t *testing.T) { require.Equal(2, len(gjson.Parse(string(bodyBytes5)).Array())) // multiple web3 req with big batch - apitypes.MaxResponseSize = 1024 * 1024 // fake max response size - request8, _ := http.NewRequest(http.MethodPost, "http://url.com", strings.NewReader(`[`+strings.Repeat(`{"jsonrpc":"2.0","method":"eth_mining","params":[],"id":1},`, 100000)+`{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":2}]`)) + apitypes.MaxResponseSize = 1024 // fake max response size + request8, _ := http.NewRequest(http.MethodPost, "http://url.com", strings.NewReader(`[`+strings.Repeat(`{"jsonrpc":"2.0","method":"eth_mining","params":[],"id":1},`, 90)+`{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":2}]`)) response8 := getServerResp(svr, request8) bodyBytes8, _ := io.ReadAll(response8.Body) require.Equal(len(bodyBytes8), 0) @@ -142,6 +142,14 @@ func TestHandlePost(t *testing.T) { response7 := getServerResp(svr, request7) bodyBytes7, _ := io.ReadAll(response7.Body) require.Contains(string(bodyBytes7), "result") + + // multiple web3 req with big batch + apitypes.MaxResponseSize = 1024 * 1024 * 100 // fake max response size + request9, _ := http.NewRequest(http.MethodPost, "http://url.com", strings.NewReader(`[`+strings.Repeat(`{"jsonrpc":"2.0","method":"eth_mining","params":[],"id":1},`, 102)+`{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":2}]`)) + response9 := getServerResp(svr, request9) + bodyBytes9, _ := io.ReadAll(response9.Body) + require.True(gjson.Valid(string(bodyBytes9))) + require.Contains(string(bodyBytes9), errMsgBatchTooLarge.Error()) } func TestGasPrice(t *testing.T) { @@ -149,7 +157,7 @@ func TestGasPrice(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().SuggestGasPrice().Return(uint64(1), nil) ret, err := web3svr.gasPrice() require.NoError(err) @@ -165,7 +173,7 @@ func TestGetChainID(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().EVMNetworkID().Return(uint32(1)) ret, err := web3svr.getChainID() require.NoError(err) @@ -177,7 +185,7 @@ func TestGetBlockNumber(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().TipHeight().Return(uint64(1)) ret, err := web3svr.getBlockNumber() require.NoError(err) @@ -189,7 +197,7 @@ func TestGetBlockByNumber(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} tsf, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -231,7 +239,7 @@ func TestGetBalance(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} balance := "111111111111111111" core.EXPECT().Account(gomock.Any()).Return(&iotextypes.AccountMeta{Balance: balance}, nil, nil) @@ -248,7 +256,7 @@ func TestGetTransactionCount(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().PendingNonce(gomock.Any()).Return(uint64(2), nil) in := gjson.Parse(`{"params":["0xDa7e12Ef57c236a06117c5e0d04a228e7181CF36", 1]}`) ret, err := web3svr.getTransactionCount(&in) @@ -261,7 +269,7 @@ func TestCall(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} t.Run("to is StakingProtocol addr", func(t *testing.T) { meta := &iotextypes.AccountMeta{ @@ -327,7 +335,7 @@ func TestEstimateGas(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().ChainID().Return(uint32(1)).Times(2) t.Run("estimate execution", func(t *testing.T) { @@ -372,7 +380,7 @@ func TestSendRawTransaction(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().EVMNetworkID().Return(uint32(1)) core.EXPECT().ChainID().Return(uint32(1)) core.EXPECT().Account(gomock.Any()).Return(&iotextypes.AccountMeta{IsContract: true}, nil, nil) @@ -389,7 +397,7 @@ func TestGetCode(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} code := "608060405234801561001057600080fd5b50610150806100206contractbytecode" data, _ := hex.DecodeString(code) core.EXPECT().Account(gomock.Any()).Return(&iotextypes.AccountMeta{ContractByteCode: data}, nil, nil) @@ -404,7 +412,7 @@ func TestGetNodeInfo(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().ServerMeta().Return("111", "", "", "222", "") ret, err := web3svr.getNodeInfo() require.NoError(err) @@ -416,7 +424,7 @@ func TestGetNetworkID(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().EVMNetworkID().Return(uint32(123)) ret, err := web3svr.getNetworkID() require.NoError(err) @@ -428,7 +436,7 @@ func TestIsSyncing(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} core.EXPECT().SyncingProgress().Return(uint64(1), uint64(2), uint64(3)) ret, err := web3svr.isSyncing() require.NoError(err) @@ -444,7 +452,7 @@ func TestGetBlockTransactionCountByHash(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} tsf, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -472,7 +480,7 @@ func TestGetBlockByHash(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} tsf, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -515,7 +523,7 @@ func TestGetTransactionByHash(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} selp, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -545,7 +553,7 @@ func TestGetLogs(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} logs := []*action.Log{ { @@ -594,7 +602,7 @@ func TestGetTransactionReceipt(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} selp, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -637,7 +645,7 @@ func TestGetBlockTransactionCountByNumber(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} tsf, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -664,7 +672,7 @@ func TestGetTransactionByBlockHashAndIndex(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} tsf, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -702,7 +710,7 @@ func TestGetTransactionByBlockNumberAndIndex(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} tsf, err := action.SignedTransfer(identityset.Address(28).String(), identityset.PrivateKey(27), uint64(1), big.NewInt(10), []byte{}, uint64(100000), big.NewInt(0)) require.NoError(err) @@ -739,7 +747,7 @@ func TestGetStorageAt(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} val := []byte("test") core.EXPECT().ReadContractStorage(gomock.Any(), gomock.Any(), gomock.Any()).Return(val, nil) @@ -754,7 +762,7 @@ func TestNewfilter(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, newAPICache(1*time.Second, "")} + web3svr := &web3Handler{core, newAPICache(1*time.Second, ""), _defaultBatchRequestLimit} ret, err := web3svr.newFilter(&filterObject{ FromBlock: "1", @@ -771,7 +779,7 @@ func TestNewBlockFilter(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, newAPICache(1*time.Second, "")} + web3svr := &web3Handler{core, newAPICache(1*time.Second, ""), _defaultBatchRequestLimit} core.EXPECT().TipHeight().Return(uint64(123)) ret, err := web3svr.newBlockFilter() @@ -784,7 +792,7 @@ func TestUninstallFilter(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, newAPICache(1*time.Second, "")} + web3svr := &web3Handler{core, newAPICache(1*time.Second, ""), _defaultBatchRequestLimit} require.NoError(web3svr.cache.Set("123456789abc", []byte("test"))) in := gjson.Parse(`{"params":["0x123456789abc"]}`) @@ -798,7 +806,7 @@ func TestGetFilterChanges(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, newAPICache(1*time.Second, "")} + web3svr := &web3Handler{core, newAPICache(1*time.Second, ""), _defaultBatchRequestLimit} core.EXPECT().TipHeight().Return(uint64(0)).Times(3) t.Run("log filterType", func(t *testing.T) { @@ -883,7 +891,7 @@ func TestGetFilterLogs(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, newAPICache(1*time.Second, "")} + web3svr := &web3Handler{core, newAPICache(1*time.Second, ""), _defaultBatchRequestLimit} logs := []*action.Log{ { @@ -930,7 +938,7 @@ func TestSubscribe(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} listener := mock_apitypes.NewMockListener(ctrl) listener.EXPECT().AddResponder(gomock.Any()).Return("streamid_1", nil).Times(2) @@ -957,7 +965,7 @@ func TestUnsubscribe(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} listener := mock_apitypes.NewMockListener(ctrl) listener.EXPECT().RemoveResponder(gomock.Any()).Return(true, nil) @@ -989,7 +997,7 @@ func TestDebugTraceTransaction(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} ctx := context.Background() tsf, err := action.SignedExecution(identityset.Address(29).String(), @@ -1020,7 +1028,7 @@ func TestDebugTraceCall(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() core := mock_apicoreservice.NewMockCoreService(ctrl) - web3svr := &web3Handler{core, nil} + web3svr := &web3Handler{core, nil, _defaultBatchRequestLimit} ctx := context.Background() tsf, err := action.SignedExecution(identityset.Address(29).String(),