fix: OOM when eth_getLogs response too large (evmos#860)

* fix: OOM when eth_getLogs response too large Closes: evmos#858 - add limit to number of logs of filter response - make block limit and log limit configurable * return error if exceeds log limit * Apply suggestions from code review * parse from config * read cli flags * add to config template * fix bloomFilter * changelog * add validation Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com> fix lint Update PR url in CHANGELOG
crypto-org-chain · Feb 15, 2022 · 7386e57 · 7386e57
1 parent 60af027
commit 7386e57
Show file tree

Hide file tree

Showing 9 changed files with 96 additions and 110 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,7 +35,11 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 # Changelog
 
-# Unreleased
+## Unreleased
+
+### Bug Fixes
+
+* (rpc) [crypto-org-chain#42](https://github.com/crypto-org-chain/ethermint/pull/42) Fix `eth_getLogs` when specify blockHash without address/topics, and limit the response size.
 
 ## [v0.7.2-cronos-6] - 2021-12-17
 

diff --git a/rpc/ethereum/backend/backend.go b/rpc/ethereum/backend/backend.go
@@ -30,7 +30,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 
-	"github.com/tharsis/ethermint/rpc/ethereum/namespaces/eth/filters"
 	"github.com/tharsis/ethermint/rpc/ethereum/types"
 	"github.com/tharsis/ethermint/server/config"
 	ethermint "github.com/tharsis/ethermint/types"
@@ -63,7 +62,6 @@ type Backend interface {
 	RPCMinGasPrice() int64
 	ChainConfig() *params.ChainConfig
 	SuggestGasTipCap() (*big.Int, error)
-	GetFilteredBlocks(from int64, to int64, filter [][]filters.BloomIV, filterAddresses bool) ([]int64, error)
 	GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock) []*evmtypes.MsgEthereumTx
 }
 
@@ -773,6 +771,16 @@ func (e *EVMBackend) RPCFilterCap() int32 {
 	return e.cfg.JSONRPC.FilterCap
 }
 
+// RPCLogsCap defines the max number of results can be returned from single `eth_getLogs` query.
+func (e *EVMBackend) RPCLogsCap() int32 {
+	return e.cfg.JSONRPC.LogsCap
+}
+
+// RPCBlockRangeCap defines the max block range allowed for `eth_getLogs` query.
+func (e *EVMBackend) RPCBlockRangeCap() int32 {
+	return e.cfg.JSONRPC.BlockRangeCap
+}
+
 // RPCMinGasPrice returns the minimum gas price for a transaction obtained from
 // the node config. If set value is 0, it will default to 20.
 
@@ -807,55 +815,6 @@ func (e *EVMBackend) SuggestGasTipCap() (*big.Int, error) {
 	return big.NewInt(1), nil
 }
 
-// GetFilteredBlocks returns the block height list match the given bloom filters.
-func (e *EVMBackend) GetFilteredBlocks(
-	from int64,
-	to int64,
-	filters [][]filters.BloomIV,
-	filterAddresses bool,
-) ([]int64, error) {
-	matchedBlocks := make([]int64, 0)
-
-BLOCKS:
-	for height := from; height <= to; height++ {
-		if err := e.ctx.Err(); err != nil {
-			e.logger.Error("EVMBackend context error", "err", err)
-			return nil, err
-		}
-
-		h := height
-		bloom, err := e.BlockBloom(&h)
-		if err != nil {
-			e.logger.Error("retrieve header failed", "blockHeight", height, "err", err)
-			return nil, err
-		}
-
-		for i, filter := range filters {
-			// filter the header bloom with the addresses
-			if filterAddresses && i == 0 {
-				if !checkMatches(bloom, filter) {
-					continue BLOCKS
-				}
-
-				// the filter doesn't have any topics
-				if len(filters) == 1 {
-					matchedBlocks = append(matchedBlocks, height)
-					continue BLOCKS
-				}
-				continue
-			}
-
-			// filter the bloom with topics
-			if len(filter) > 0 && !checkMatches(bloom, filter) {
-				continue BLOCKS
-			}
-		}
-		matchedBlocks = append(matchedBlocks, height)
-	}
-
-	return matchedBlocks, nil
-}
-
 // GetEthereumMsgsFromTendermintBlock returns all real MsgEthereumTxs from a Tendermint block.
 // It also ensures consistency over the correct txs indexes across RPC endpoints
 func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.ResultBlock) []*evmtypes.MsgEthereumTx {
@@ -888,16 +847,3 @@ func (e *EVMBackend) GetEthereumMsgsFromTendermintBlock(block *tmrpctypes.Result
 
 	return result
 }
-
-// checkMatches revised the function from
-// https://github.com/ethereum/go-ethereum/blob/401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5/core/types/bloom9.go#L88
-func checkMatches(bloom ethtypes.Bloom, filter []filters.BloomIV) bool {
-	for _, bloomIV := range filter {
-		if bloomIV.V[0] == bloomIV.V[0]&bloom[bloomIV.I[0]] &&
-			bloomIV.V[1] == bloomIV.V[1]&bloom[bloomIV.I[1]] &&
-			bloomIV.V[2] == bloomIV.V[2]&bloom[bloomIV.I[2]] {
-			return true
-		}
-	}
-	return false
-}
diff --git a/rpc/ethereum/namespaces/eth/filters/api.go b/rpc/ethereum/namespaces/eth/filters/api.go
@@ -29,13 +29,14 @@ type Backend interface {
 	HeaderByHash(blockHash common.Hash) (*ethtypes.Header, error)
 	GetLogs(blockHash common.Hash) ([][]*ethtypes.Log, error)
 	GetLogsByNumber(blockNum types.BlockNumber) ([][]*ethtypes.Log, error)
+	BlockBloom(height *int64) (ethtypes.Bloom, error)
 
 	GetTransactionLogs(txHash common.Hash) ([]*ethtypes.Log, error)
 	BloomStatus() (uint64, uint64)
 
-	GetFilteredBlocks(from int64, to int64, bloomIndexes [][]BloomIV, filterAddresses bool) ([]int64, error)
-
 	RPCFilterCap() int32
+	RPCLogsCap() int32
+	RPCBlockRangeCap() int32
 }
 
 // consider a filter inactive if it has not been polled for within deadline
@@ -494,7 +495,7 @@ func (api *PublicFilterAPI) GetLogs(ctx context.Context, crit filters.FilterCrit
 	}
 
 	// Run the filter and return all the logs
-	logs, err := filter.Logs(ctx)
+	logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap()))
 	if err != nil {
 		return nil, err
 	}
@@ -555,7 +556,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*et
 		filter = NewRangeFilter(api.logger, api.backend, begin, end, f.crit.Addresses, f.crit.Topics)
 	}
 	// Run the filter and return all the logs
-	logs, err := filter.Logs(ctx)
+	logs, err := filter.Logs(ctx, int(api.backend.RPCLogsCap()), int64(api.backend.RPCBlockRangeCap()))
 	if err != nil {
 		return nil, err
 	}

diff --git a/rpc/ethereum/namespaces/eth/filters/filters.go b/rpc/ethereum/namespaces/eth/filters/filters.go
@@ -84,13 +84,12 @@ func newFilter(logger log.Logger, backend Backend, criteria filters.FilterCriter
 }
 
 const (
-	maxFilterBlocks = 100000
-	maxToOverhang   = 600
+	maxToOverhang = 600
 )
 
 // Logs searches the blockchain for matching log entries, returning all from the
 // first block that contains matches, updating the start of the filter accordingly.
-func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
+func (f *Filter) Logs(_ context.Context, logLimit int, blockLimit int64) ([]*ethtypes.Log, error) {
 	logs := []*ethtypes.Log{}
 	var err error
 
@@ -105,7 +104,7 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
 			return nil, errors.Errorf("unknown block header %s", f.criteria.BlockHash.String())
 		}
 
-		return f.blockLogs(header)
+		return f.blockLogs(header.Number.Int64(), header.Bloom)
 	}
 
 	// Figure out the limits of the filter range
@@ -131,8 +130,8 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
 		f.criteria.ToBlock = big.NewInt(1)
 	}
 
-	if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > maxFilterBlocks {
-		return nil, errors.Errorf("maximum [from, to] blocks distance: %d", maxFilterBlocks)
+	if f.criteria.ToBlock.Int64()-f.criteria.FromBlock.Int64() > blockLimit {
+		return nil, errors.Errorf("maximum [from, to] blocks distance: %d", blockLimit)
 	}
 
 	// check bounds
@@ -145,36 +144,38 @@ func (f *Filter) Logs(_ context.Context) ([]*ethtypes.Log, error) {
 	from := f.criteria.FromBlock.Int64()
 	to := f.criteria.ToBlock.Int64()
 
-	blocks, err := f.backend.GetFilteredBlocks(from, to, f.bloomFilters, len(f.criteria.Addresses) > 0)
-	if err != nil {
-		return nil, err
-	}
+	for i := from; i <= to; i++ {
+		height := i
+		bloom, err := f.backend.BlockBloom(&height)
+		if err != nil {
+			return nil, err
+		}
 
-	for _, height := range blocks {
-		ethLogs, err := f.backend.GetLogsByNumber(types.BlockNumber(height))
+		filtered, err := f.blockLogs(height, bloom)
 		if err != nil {
-			return logs, errors.Wrapf(err, "failed to fetch block by number %d", height)
+			return nil, errors.Wrapf(err, "failed to fetch block by number %d", height)
 		}
 
-		for _, ethLog := range ethLogs {
-			filtered := FilterLogs(ethLog, f.criteria.FromBlock, f.criteria.ToBlock, f.criteria.Addresses, f.criteria.Topics)
-			logs = append(logs, filtered...)
+		// check logs limit
+		if len(logs)+len(filtered) > logLimit {
+			return nil, errors.Errorf("query returned more than %d results", logLimit)
 		}
+		logs = append(logs, filtered...)
 	}
 	return logs, nil
 }
 
 // blockLogs returns the logs matching the filter criteria within a single block.
-func (f *Filter) blockLogs(header *ethtypes.Header) ([]*ethtypes.Log, error) {
-	if !bloomFilter(header.Bloom, f.criteria.Addresses, f.criteria.Topics) {
+func (f *Filter) blockLogs(height int64, bloom ethtypes.Bloom) ([]*ethtypes.Log, error) {
+	if !bloomFilter(bloom, f.criteria.Addresses, f.criteria.Topics) {
 		return []*ethtypes.Log{}, nil
 	}
 
 	// DANGER: do not call GetLogs(header.Hash())
 	// eth header's hash doesn't match tm block hash
-	logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(header.Number.Int64()))
+	logsList, err := f.backend.GetLogsByNumber(types.BlockNumber(height))
 	if err != nil {
-		return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", header.Number.Int64())
+		return []*ethtypes.Log{}, errors.Wrapf(err, "failed to fetch logs block number %d", height)
 	}
 
 	var unfiltered []*ethtypes.Log

diff --git a/rpc/ethereum/namespaces/eth/filters/utils.go b/rpc/ethereum/namespaces/eth/filters/utils.go
@@ -57,9 +57,10 @@ func includes(addresses []common.Address, a common.Address) bool {
 	return false
 }
 
+// https://github.com/ethereum/go-ethereum/blob/v1.10.14/eth/filters/filter.go#L321
 func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]common.Hash) bool {
-	var included bool
 	if len(addresses) > 0 {
+		var included bool
 		for _, addr := range addresses {
 			if ethtypes.BloomLookup(bloom, addr) {
 				included = true
@@ -72,15 +73,18 @@ func bloomFilter(bloom ethtypes.Bloom, addresses []common.Address, topics [][]co
 	}
 
 	for _, sub := range topics {
-		included = len(sub) == 0 // empty rule set == wildcard
+		included := len(sub) == 0 // empty rule set == wildcard
 		for _, topic := range sub {
 			if ethtypes.BloomLookup(bloom, topic) {
 				included = true
 				break
 			}
 		}
+		if !included {
+			return false
+		}
 	}
-	return included
+	return true
 }
 
 // returnHashes is a helper that will return an empty hash array case the given hash array is nil,

diff --git a/server/config/config.go b/server/config/config.go
@@ -30,6 +30,10 @@ const (
 	DefaultGasCap uint64 = 25000000
 
 	DefaultFilterCap int32 = 200
+
+	DefaultLogsCap int32 = 10000
+
+	DefaultBlockRangeCap int32 = 10000
 )
 
 var evmTracers = []string{"json", "markdown", "struct", "access_list"}
@@ -65,6 +69,10 @@ type JSONRPCConfig struct {
 	FilterCap int32 `mapstructure:"filter-cap"`
 	// Enable defines if the EVM RPC server should be enabled.
 	Enable bool `mapstructure:"enable"`
+	// LogsCap defines the max number of results can be returned from single `eth_getLogs` query.
+	LogsCap int32 `mapstructure:"logs-cap"`
+	// BlockRangeCap defines the max block range allowed for `eth_getLogs` query.
+	BlockRangeCap int32 `mapstructure:"block-range-cap"`
 }
 
 // TLSConfig defines the certificate and matching private key for the server.
@@ -144,12 +152,14 @@ func GetDefaultAPINamespaces() []string {
 // DefaultJSONRPCConfig returns an EVM config with the JSON-RPC API enabled by default
 func DefaultJSONRPCConfig() *JSONRPCConfig {
 	return &JSONRPCConfig{
-		Enable:    true,
-		API:       GetDefaultAPINamespaces(),
-		Address:   DefaultJSONRPCAddress,
-		WsAddress: DefaultJSONRPCWsAddress,
-		GasCap:    DefaultGasCap,
-		FilterCap: DefaultFilterCap,
+		Enable:        true,
+		API:           GetDefaultAPINamespaces(),
+		Address:       DefaultJSONRPCAddress,
+		WsAddress:     DefaultJSONRPCWsAddress,
+		GasCap:        DefaultGasCap,
+		FilterCap:     DefaultFilterCap,
+		BlockRangeCap: DefaultBlockRangeCap,
+		LogsCap:       DefaultLogsCap,
 	}
 }
 
@@ -163,6 +173,14 @@ func (c JSONRPCConfig) Validate() error {
 		return errors.New("JSON-RPC filter-cap cannot be negative")
 	}
 
+	if c.LogsCap < 0 {
+		return errors.New("JSON-RPC logs cap cannot be negative")
+	}
+
+	if c.BlockRangeCap < 0 {
+		return errors.New("JSON-RPC block range cap cannot be negative")
+	}
+
 	// TODO: validate APIs
 	seenAPIs := make(map[string]bool)
 	for _, api := range c.API {
@@ -211,12 +229,14 @@ func GetConfig(v *viper.Viper) Config {
 			Tracer: v.GetString("evm.tracer"),
 		},
 		JSONRPC: JSONRPCConfig{
-			Enable:    v.GetBool("json-rpc.enable"),
-			API:       v.GetStringSlice("json-rpc.api"),
-			Address:   v.GetString("json-rpc.address"),
-			WsAddress: v.GetString("json-rpc.ws-address"),
-			GasCap:    v.GetUint64("json-rpc.gas-cap"),
-			FilterCap: v.GetInt32("json-rpc.filter-cap"),
+			Enable:        v.GetBool("json-rpc.enable"),
+			API:           v.GetStringSlice("json-rpc.api"),
+			Address:       v.GetString("json-rpc.address"),
+			WsAddress:     v.GetString("json-rpc.ws-address"),
+			GasCap:        v.GetUint64("json-rpc.gas-cap"),
+			FilterCap:     v.GetInt32("json-rpc.filter-cap"),
+			LogsCap:       v.GetInt32("json-rpc.logs-cap"),
+			BlockRangeCap: v.GetInt32("json-rpc.block-range-cap"),
 		},
 		TLS: TLSConfig{
 			CertificatePath: v.GetString("tls.certificate-path"),

diff --git a/server/config/toml.go b/server/config/toml.go
@@ -38,6 +38,12 @@ gas-cap = {{ .JSONRPC.GasCap }}
 # FilterCap sets the global cap for total number of filters that can be created
 filter-cap = {{ .JSONRPC.FilterCap }}
 
+# LogsCap defines the max number of results can be returned from single 'eth_getLogs' query.
+logs-cap = {{ .JSONRPC.LogsCap }}
+
+# BlockRangeCap defines the max block range allowed for 'eth_getLogs' query.
+block-range-cap = {{ .JSONRPC.BlockRangeCap }}
+
 ###############################################################################
 ###                             TLS Configuration                           ###
 ###############################################################################

diff --git a/server/flags/flags.go b/server/flags/flags.go
@@ -26,12 +26,14 @@ const (
 
 // JSON-RPC flags
 const (
-	JSONRPCEnable    = "json-rpc.enable"
-	JSONRPCAPI       = "json-rpc.api"
-	JSONRPCAddress   = "json-rpc.address"
-	JSONWsAddress    = "json-rpc.ws-address"
-	JSONRPCGasCap    = "json-rpc.gas-cap"
-	JSONRPCFilterCap = "json-rpc.filter-cap"
+	JSONRPCEnable        = "json-rpc.enable"
+	JSONRPCAPI           = "json-rpc.api"
+	JSONRPCAddress       = "json-rpc.address"
+	JSONWsAddress        = "json-rpc.ws-address"
+	JSONRPCGasCap        = "json-rpc.gas-cap"
+	JSONRPCFilterCap     = "json-rpc.filter-cap"
+	JSONRPCLogsCap       = "json-rpc.logs-cap"
+	JSONRPCBlockRangeCap = "json-rpc.block-range-cap"
 )
 
 // EVM flags

diff --git a/server/start.go b/server/start.go
@@ -157,6 +157,8 @@ which accepts a path for the resulting pprof file.
 	cmd.Flags().String(srvflags.JSONWsAddress, config.DefaultJSONRPCWsAddress, "the JSON-RPC WS server address to listen on")
 	cmd.Flags().Uint64(srvflags.JSONRPCGasCap, config.DefaultGasCap, "Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite)")
 	cmd.Flags().Int32(srvflags.JSONRPCFilterCap, config.DefaultFilterCap, "Sets the global cap for total number of filters that can be created")
+	cmd.Flags().Int32(srvflags.JSONRPCLogsCap, config.DefaultLogsCap, "Sets the max number of results can be returned from single `eth_getLogs` query")
+	cmd.Flags().Int32(srvflags.JSONRPCBlockRangeCap, config.DefaultBlockRangeCap, "Sets the max block range allowed for `eth_getLogs` query")
 
 	cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)")