From 3d5f5e620483c56d85188b82f85fff1fbe623bf0 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Thu, 19 Aug 2021 19:53:36 -0400 Subject: [PATCH 01/17] revise the way of the logs store into the stateDB --- x/evm/keeper/grpc_query.go | 19 +++++-- x/evm/keeper/keeper.go | 102 ++++++++++++++++++++++++++++++++----- x/evm/keeper/statedb.go | 8 +-- x/evm/types/key.go | 2 + 4 files changed, 108 insertions(+), 23 deletions(-) diff --git a/x/evm/keeper/grpc_query.go b/x/evm/keeper/grpc_query.go index b097ffbeba..a487b3914b 100644 --- a/x/evm/keeper/grpc_query.go +++ b/x/evm/keeper/grpc_query.go @@ -232,15 +232,19 @@ func (k Keeper) BlockLogs(c context.Context, req *types.QueryBlockLogsRequest) ( ctx := sdk.UnwrapSDKContext(c) store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) - txLogs := []types.TransactionLogs{} + logs := make(map[string][]*types.Log) pageRes, err := query.FilteredPaginate(store, req.Pagination, func(_, value []byte, accumulate bool) (bool, error) { - var txLog types.TransactionLogs + var txLog types.Log k.cdc.MustUnmarshal(value, &txLog) - if len(txLog.Logs) > 0 && txLog.Logs[0].BlockHash == req.Hash { + if txLog.BlockHash == req.Hash { if accumulate { - txLogs = append(txLogs, txLog) + if logs[txLog.TxHash] == nil { + logs[txLog.TxHash] = []*types.Log{} + } + + logs[txLog.TxHash] = append(logs[txLog.TxHash], &txLog) } return true, nil } @@ -252,8 +256,13 @@ func (k Keeper) BlockLogs(c context.Context, req *types.QueryBlockLogsRequest) ( return nil, status.Error(codes.Internal, err.Error()) } + var txsLogs = []types.TransactionLogs{} + for hash, log := range logs { + txsLogs = append(txsLogs, types.TransactionLogs{Hash: hash, Logs: log}) + } + return &types.QueryBlockLogsResponse{ - TxLogs: txLogs, + TxLogs: txsLogs, Pagination: pageRes, }, nil } diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index 7465a215d1..d81858c114 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -229,16 +229,36 @@ func (k Keeper) ResetRefundTransient(ctx sdk.Context) { // GetAllTxLogs return all the transaction logs from the store. func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { store := ctx.KVStore(k.storeKey) - iterator := sdk.KVStorePrefixIterator(store, types.KeyPrefixLogs) + iterator := sdk.KVStorePrefixIterator(store, types.KeyPrefixTxLogNum) defer iterator.Close() txsLogs := []types.TransactionLogs{} for ; iterator.Valid(); iterator.Next() { - var txLog types.TransactionLogs - k.cdc.MustUnmarshal(iterator.Value(), &txLog) + logNum := new(big.Int).SetBytes(iterator.Value()) + txHash := iterator.Key() + + var txlogs types.TransactionLogs + txlogs.Hash = common.Bytes2Hex(txHash) + + s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + for i := int64(0); i < logNum.Int64(); i++ { + var key []byte + key = append(key, txHash...) + key = append(key, new(big.Int).SetInt64(i).Bytes()...) + + logbz := s.Get(key) + if logbz == nil { + // TODO: error handle? + continue + } + + var txLog types.Log + k.cdc.MustUnmarshal(logbz, &txLog) + txlogs.Logs = append(txlogs.Logs, &txLog) + } // add a new entry - txsLogs = append(txsLogs, txLog) + txsLogs = append(txsLogs, txlogs) } return txsLogs } @@ -246,33 +266,87 @@ func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { // GetLogs returns the current logs for a given transaction hash from the KVStore. // This function returns an empty, non-nil slice if no logs are found. func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { - store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogNum) bz := store.Get(txHash.Bytes()) - if len(bz) == 0 { + if bz == nil { + return []*ethtypes.Log{} + } + + logNum := new(big.Int).SetBytes(bz) + if logNum.Int64() == 0 { return []*ethtypes.Log{} } - var logs types.TransactionLogs - k.cdc.MustUnmarshal(bz, &logs) + store = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + var logs []*ethtypes.Log + for i := 0; i < int(logNum.Int64()); i++ { + var key []byte + key = append(key, txHash.Bytes()...) + + key = append(key, new(big.Int).SetInt64(int64(i)).Bytes()...) + logbz := store.Get(key) - return logs.EthLogs() + if logbz == nil { + // TODO: how to handle this? + continue + } + + var log types.Log + k.cdc.MustUnmarshal(logbz, &log) + + logs = append(logs, log.ToEthereum()) + } + return logs } // SetLogs sets the logs for a transaction in the KVStore. func (k Keeper) SetLogs(txHash common.Hash, logs []*ethtypes.Log) { - store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + var s prefix.Store + len := len(logs) + if len > 0 { + s = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogNum) + s.Set(txHash.Bytes(), new(big.Int).SetInt64(int64(len)).Bytes()) + } - txLogs := types.NewTransactionLogsFromEth(txHash, logs) - bz := k.cdc.MustMarshal(&txLogs) + s = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + for index, log := range logs { + k.SetLog(txHash, log, s, int64(index)) + } +} + +// SetLog sets the log for a transaction in the KVStore. +func (k Keeper) SetLog(txHash common.Hash, log *ethtypes.Log, s prefix.Store, index int64) { + txIndexLog := types.NewLogFromEth(log) + bz := k.cdc.MustMarshal(txIndexLog) - store.Set(txHash.Bytes(), bz) + var key []byte + key = append(key, txHash.Bytes()...) + key = append(key, new(big.Int).SetInt64(index).Bytes()...) + s.Set(key, bz) } // DeleteLogs removes the logs from the KVStore. It is used during journal.Revert. func (k Keeper) DeleteTxLogs(ctx sdk.Context, txHash common.Hash) { - store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) + store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixTxLogNum) + + b := store.Get(txHash.Bytes()) + if b == nil { + return + } + + txLogCnt := new(big.Int).SetBytes(b) + store.Delete(txHash.Bytes()) + + store = prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) + for i := int64(0); i < txLogCnt.Int64(); i++ { + var key []byte + key = append(key, txHash.Bytes()...) + key = append(key, new(big.Int).SetInt64(i).Bytes()...) + + store.Delete(key) + } } // GetLogSizeTransient returns EVM log index on the current block. diff --git a/x/evm/keeper/statedb.go b/x/evm/keeper/statedb.go index 21eaf91281..56f12943ee 100644 --- a/x/evm/keeper/statedb.go +++ b/x/evm/keeper/statedb.go @@ -608,10 +608,10 @@ func (k *Keeper) AddLog(log *ethtypes.Log) { log.Index = uint(k.GetLogSizeTransient()) k.IncreaseLogSizeTransient() - logs := k.GetTxLogs(log.TxHash) - logs = append(logs, log) - - k.SetLogs(log.TxHash, logs) + s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogNum) + bi := new(big.Int).SetBytes(s.Get(log.TxHash.Bytes())) + s.Set(log.TxHash.Bytes(), new(big.Int).SetInt64(bi.Int64()+1).Bytes()) + k.SetLog(log.TxHash, log, prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs), bi.Int64()) k.Logger(k.Ctx()).Debug( "log added", diff --git a/x/evm/types/key.go b/x/evm/types/key.go index 02de5939cb..1c056b94bb 100644 --- a/x/evm/types/key.go +++ b/x/evm/types/key.go @@ -31,6 +31,7 @@ const ( prefixLogs prefixCode prefixStorage + prefixTxLogNum ) // prefix bytes for the EVM transient store @@ -52,6 +53,7 @@ var ( KeyPrefixLogs = []byte{prefixLogs} KeyPrefixCode = []byte{prefixCode} KeyPrefixStorage = []byte{prefixStorage} + KeyPrefixTxLogNum = []byte{prefixTxLogNum} ) // Transient Store key prefixes From 5de2e3b6488ef90dc7cfc7b1bda82674eba1ec44 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 11:30:11 -0400 Subject: [PATCH 02/17] wording --- x/evm/keeper/keeper.go | 22 +++++++++++----------- x/evm/keeper/statedb.go | 2 +- x/evm/types/key.go | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index d81858c114..c87e43f5d2 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -229,19 +229,19 @@ func (k Keeper) ResetRefundTransient(ctx sdk.Context) { // GetAllTxLogs return all the transaction logs from the store. func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { store := ctx.KVStore(k.storeKey) - iterator := sdk.KVStorePrefixIterator(store, types.KeyPrefixTxLogNum) + iterator := sdk.KVStorePrefixIterator(store, types.KeyPrefixTxLogCount) defer iterator.Close() txsLogs := []types.TransactionLogs{} for ; iterator.Valid(); iterator.Next() { - logNum := new(big.Int).SetBytes(iterator.Value()) + txLogCount := new(big.Int).SetBytes(iterator.Value()) txHash := iterator.Key() var txlogs types.TransactionLogs txlogs.Hash = common.Bytes2Hex(txHash) s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - for i := int64(0); i < logNum.Int64(); i++ { + for i := int64(0); i < txLogCount.Int64(); i++ { var key []byte key = append(key, txHash...) key = append(key, new(big.Int).SetInt64(i).Bytes()...) @@ -266,21 +266,21 @@ func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { // GetLogs returns the current logs for a given transaction hash from the KVStore. // This function returns an empty, non-nil slice if no logs are found. func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { - store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogNum) + store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) bz := store.Get(txHash.Bytes()) if bz == nil { return []*ethtypes.Log{} } - logNum := new(big.Int).SetBytes(bz) - if logNum.Int64() == 0 { + txLogCount := new(big.Int).SetBytes(bz) + if txLogCount.Int64() == 0 { return []*ethtypes.Log{} } store = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) var logs []*ethtypes.Log - for i := 0; i < int(logNum.Int64()); i++ { + for i := 0; i < int(txLogCount.Int64()); i++ { var key []byte key = append(key, txHash.Bytes()...) @@ -305,7 +305,7 @@ func (k Keeper) SetLogs(txHash common.Hash, logs []*ethtypes.Log) { var s prefix.Store len := len(logs) if len > 0 { - s = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogNum) + s = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) s.Set(txHash.Bytes(), new(big.Int).SetInt64(int64(len)).Bytes()) } @@ -328,19 +328,19 @@ func (k Keeper) SetLog(txHash common.Hash, log *ethtypes.Log, s prefix.Store, in // DeleteLogs removes the logs from the KVStore. It is used during journal.Revert. func (k Keeper) DeleteTxLogs(ctx sdk.Context, txHash common.Hash) { - store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixTxLogNum) + store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixTxLogCount) b := store.Get(txHash.Bytes()) if b == nil { return } - txLogCnt := new(big.Int).SetBytes(b) + txLogCount := new(big.Int).SetBytes(b) store.Delete(txHash.Bytes()) store = prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) - for i := int64(0); i < txLogCnt.Int64(); i++ { + for i := int64(0); i < txLogCount.Int64(); i++ { var key []byte key = append(key, txHash.Bytes()...) key = append(key, new(big.Int).SetInt64(i).Bytes()...) diff --git a/x/evm/keeper/statedb.go b/x/evm/keeper/statedb.go index 56f12943ee..0b21567fb7 100644 --- a/x/evm/keeper/statedb.go +++ b/x/evm/keeper/statedb.go @@ -608,7 +608,7 @@ func (k *Keeper) AddLog(log *ethtypes.Log) { log.Index = uint(k.GetLogSizeTransient()) k.IncreaseLogSizeTransient() - s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogNum) + s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) bi := new(big.Int).SetBytes(s.Get(log.TxHash.Bytes())) s.Set(log.TxHash.Bytes(), new(big.Int).SetInt64(bi.Int64()+1).Bytes()) k.SetLog(log.TxHash, log, prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs), bi.Int64()) diff --git a/x/evm/types/key.go b/x/evm/types/key.go index 1c056b94bb..29846c5fe7 100644 --- a/x/evm/types/key.go +++ b/x/evm/types/key.go @@ -31,7 +31,7 @@ const ( prefixLogs prefixCode prefixStorage - prefixTxLogNum + prefixTxLogCount ) // prefix bytes for the EVM transient store @@ -53,7 +53,7 @@ var ( KeyPrefixLogs = []byte{prefixLogs} KeyPrefixCode = []byte{prefixCode} KeyPrefixStorage = []byte{prefixStorage} - KeyPrefixTxLogNum = []byte{prefixTxLogNum} + KeyPrefixTxLogCount = []byte{prefixTxLogCount} ) // Transient Store key prefixes From a426b35df151c0852e29747ce3c5f777c6180db4 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 11:36:22 -0400 Subject: [PATCH 03/17] refine GetTxLogs/SetLogs --- x/evm/keeper/keeper.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index c87e43f5d2..3390364a63 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -269,17 +269,14 @@ func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) bz := store.Get(txHash.Bytes()) - if bz == nil { + if len(bz) == 0 { return []*ethtypes.Log{} } + store = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) txLogCount := new(big.Int).SetBytes(bz) - if txLogCount.Int64() == 0 { - return []*ethtypes.Log{} - } - store = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - var logs []*ethtypes.Log + logs := []*ethtypes.Log{} for i := 0; i < int(txLogCount.Int64()); i++ { var key []byte key = append(key, txHash.Bytes()...) @@ -302,16 +299,16 @@ func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { // SetLogs sets the logs for a transaction in the KVStore. func (k Keeper) SetLogs(txHash common.Hash, logs []*ethtypes.Log) { - var s prefix.Store + var logStore prefix.Store len := len(logs) if len > 0 { - s = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) + s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) s.Set(txHash.Bytes(), new(big.Int).SetInt64(int64(len)).Bytes()) + logStore = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) } - s = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) for index, log := range logs { - k.SetLog(txHash, log, s, int64(index)) + k.SetLog(txHash, log, logStore, int64(index)) } } From 14aabd14e1544e5ab8229f035cf7b442ae4b793f Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 11:45:41 -0400 Subject: [PATCH 04/17] nit --- x/evm/keeper/grpc_query.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/x/evm/keeper/grpc_query.go b/x/evm/keeper/grpc_query.go index a487b3914b..5ae36d0e77 100644 --- a/x/evm/keeper/grpc_query.go +++ b/x/evm/keeper/grpc_query.go @@ -240,10 +240,6 @@ func (k Keeper) BlockLogs(c context.Context, req *types.QueryBlockLogsRequest) ( if txLog.BlockHash == req.Hash { if accumulate { - if logs[txLog.TxHash] == nil { - logs[txLog.TxHash] = []*types.Log{} - } - logs[txLog.TxHash] = append(logs[txLog.TxHash], &txLog) } return true, nil From 118d95daf055692292948c4e8546893a5b3e4223 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 12:32:01 -0400 Subject: [PATCH 05/17] add bytes_uint16 convert helper --- x/evm/types/utils.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x/evm/types/utils.go b/x/evm/types/utils.go index 44ee600a7a..0864348d38 100644 --- a/x/evm/types/utils.go +++ b/x/evm/types/utils.go @@ -1,6 +1,7 @@ package types import ( + "encoding/binary" "fmt" "github.com/gogo/protobuf/proto" @@ -87,3 +88,28 @@ func BinSearch(lo uint64, hi uint64, executable func(uint64) (bool, *MsgEthereum } return hi, nil } + +// Uint16ToBytes transfer value to variable size []byte +func Uint16ToBytes(v uint16) []byte { + bz := []byte{0x00, 0x00} + binary.BigEndian.PutUint16(bz, v) + + if v < 256 { + return bz[1:] + } + + return bz +} + +// BytesToUint16 transfer []byte to uint16. +// Note: If the bytes length greater than two, the value lower than 256 will be returned. +func BytesToUint16(bz []byte) uint16 { + l := len(bz) + if l == 0 { + return 0 + } else if l == 1 { + return binary.BigEndian.Uint16([]byte{0x00, bz[0]}) + } + + return binary.BigEndian.Uint16(bz) +} From c7f80e5c6f46abe7d8cd1adc42ea8b8c6b00d6e8 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 12:55:11 -0400 Subject: [PATCH 06/17] using helper to instead of big.Int --- x/evm/keeper/keeper.go | 37 +++++++++++++++++++------------------ x/evm/keeper/statedb.go | 14 +++++++++++--- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index 3390364a63..f6124275d7 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -234,18 +234,17 @@ func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { txsLogs := []types.TransactionLogs{} for ; iterator.Valid(); iterator.Next() { - txLogCount := new(big.Int).SetBytes(iterator.Value()) + txLogCount := types.BytesToUint16(iterator.Value()) txHash := iterator.Key() var txlogs types.TransactionLogs txlogs.Hash = common.Bytes2Hex(txHash) s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - for i := int64(0); i < txLogCount.Int64(); i++ { + for i := uint16(0); i < txLogCount; i++ { var key []byte key = append(key, txHash...) - key = append(key, new(big.Int).SetInt64(i).Bytes()...) - + key = append(key, types.Uint16ToBytes(i)...) logbz := s.Get(key) if logbz == nil { // TODO: error handle? @@ -274,14 +273,13 @@ func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { } store = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - txLogCount := new(big.Int).SetBytes(bz) + txLogCount := types.BytesToUint16(bz) logs := []*ethtypes.Log{} - for i := 0; i < int(txLogCount.Int64()); i++ { + for i := uint16(0); i < txLogCount; i++ { var key []byte key = append(key, txHash.Bytes()...) - - key = append(key, new(big.Int).SetInt64(int64(i)).Bytes()...) + key = append(key, types.Uint16ToBytes(i)...) logbz := store.Get(key) if logbz == nil { @@ -301,25 +299,29 @@ func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { func (k Keeper) SetLogs(txHash common.Hash, logs []*ethtypes.Log) { var logStore prefix.Store len := len(logs) - if len > 0 { + + if len > 65535 { + //TODO: to log a warning, but still allows to store the logs? + } else if len > 0 { s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) - s.Set(txHash.Bytes(), new(big.Int).SetInt64(int64(len)).Bytes()) + + s.Set(txHash.Bytes(), types.Uint16ToBytes(uint16(len))) logStore = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) } for index, log := range logs { - k.SetLog(txHash, log, logStore, int64(index)) + k.SetLog(txHash, log, logStore, uint16(index)) } } // SetLog sets the log for a transaction in the KVStore. -func (k Keeper) SetLog(txHash common.Hash, log *ethtypes.Log, s prefix.Store, index int64) { +func (k Keeper) SetLog(txHash common.Hash, log *ethtypes.Log, s prefix.Store, index uint16) { txIndexLog := types.NewLogFromEth(log) bz := k.cdc.MustMarshal(txIndexLog) var key []byte key = append(key, txHash.Bytes()...) - key = append(key, new(big.Int).SetInt64(index).Bytes()...) + key = append(key, types.Uint16ToBytes(index)...) s.Set(key, bz) } @@ -328,19 +330,18 @@ func (k Keeper) DeleteTxLogs(ctx sdk.Context, txHash common.Hash) { store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixTxLogCount) b := store.Get(txHash.Bytes()) - if b == nil { + if len(b) == 0 { return } - txLogCount := new(big.Int).SetBytes(b) - + txLogCount := types.BytesToUint16(b) store.Delete(txHash.Bytes()) store = prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) - for i := int64(0); i < txLogCount.Int64(); i++ { + for i := uint16(0); i < txLogCount; i++ { var key []byte key = append(key, txHash.Bytes()...) - key = append(key, new(big.Int).SetInt64(i).Bytes()...) + key = append(key, types.Uint16ToBytes(i)...) store.Delete(key) } diff --git a/x/evm/keeper/statedb.go b/x/evm/keeper/statedb.go index 0b21567fb7..26ac0d96ff 100644 --- a/x/evm/keeper/statedb.go +++ b/x/evm/keeper/statedb.go @@ -609,9 +609,17 @@ func (k *Keeper) AddLog(log *ethtypes.Log) { k.IncreaseLogSizeTransient() s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) - bi := new(big.Int).SetBytes(s.Get(log.TxHash.Bytes())) - s.Set(log.TxHash.Bytes(), new(big.Int).SetInt64(bi.Int64()+1).Bytes()) - k.SetLog(log.TxHash, log, prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs), bi.Int64()) + txLogCount := types.BytesToUint16(s.Get(log.TxHash.Bytes())) + if txLogCount == 65535 { + k.Logger(k.Ctx()).Error( + "failed to add log due to the log size limit", + "tx-hash-ethereum", log.TxHash.Hex(), + "log-index", int(log.Index), + ) + return + } + s.Set(log.TxHash.Bytes(), types.Uint16ToBytes(txLogCount+1)) + k.SetLog(log.TxHash, log, prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs), txLogCount) k.Logger(k.Ctx()).Debug( "log added", From 3ac596660af63b7d1fdf3dcf65154614a8f7204b Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 13:03:00 -0400 Subject: [PATCH 07/17] nit --- x/evm/types/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x/evm/types/utils.go b/x/evm/types/utils.go index 0864348d38..5a780f91e9 100644 --- a/x/evm/types/utils.go +++ b/x/evm/types/utils.go @@ -108,7 +108,7 @@ func BytesToUint16(bz []byte) uint16 { if l == 0 { return 0 } else if l == 1 { - return binary.BigEndian.Uint16([]byte{0x00, bz[0]}) + return uint16(bz[0]) } return binary.BigEndian.Uint16(bz) From 4d1c6a027ba30b754c9bee529c63ed9911eb92f2 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 14:08:52 -0400 Subject: [PATCH 08/17] helper tests and fixes --- x/evm/types/utils.go | 2 +- x/evm/types/utils_test.go | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/x/evm/types/utils.go b/x/evm/types/utils.go index 5a780f91e9..43b101ad78 100644 --- a/x/evm/types/utils.go +++ b/x/evm/types/utils.go @@ -111,5 +111,5 @@ func BytesToUint16(bz []byte) uint16 { return uint16(bz[0]) } - return binary.BigEndian.Uint16(bz) + return binary.BigEndian.Uint16(bz[l-2:]) } diff --git a/x/evm/types/utils_test.go b/x/evm/types/utils_test.go index b1fa74b95c..dcaf83e749 100644 --- a/x/evm/types/utils_test.go +++ b/x/evm/types/utils_test.go @@ -98,3 +98,24 @@ func TestBinSearch(t *testing.T) { require.Error(t, err) require.Equal(t, gas, uint64(0)) } + +func TestUint16ToBytes(t *testing.T) { + require.Equal(t, []byte{0x00}, evmtypes.Uint16ToBytes(0)) + require.Equal(t, []byte{0x01}, evmtypes.Uint16ToBytes(1)) + require.Equal(t, []byte{0xff}, evmtypes.Uint16ToBytes(255)) + require.Equal(t, []byte{0x01, 0x00}, evmtypes.Uint16ToBytes(256)) + require.Equal(t, []byte{0xff, 0xff}, evmtypes.Uint16ToBytes(65535)) +} + +func TestBytesToUint16(t *testing.T) { + require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{})) + require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{0x00})) + require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{0x00, 0x00})) + require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{0x01, 0x00, 0x00})) + require.Equal(t, uint16(1), evmtypes.BytesToUint16([]byte{0x01})) + require.Equal(t, uint16(1), evmtypes.BytesToUint16([]byte{0x01, 0x00, 0x01})) + require.Equal(t, uint16(255), evmtypes.BytesToUint16([]byte{0xff})) + require.Equal(t, uint16(256), evmtypes.BytesToUint16([]byte{0x01, 0x00})) + require.Equal(t, uint16(65535), evmtypes.BytesToUint16([]byte{0xff, 0xff})) + require.Equal(t, uint16(65535), evmtypes.BytesToUint16([]byte{0x01, 0xff, 0xff})) +} From e65384b13081590f3be95008a9334cca56dc41db Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 20 Aug 2021 14:25:43 -0400 Subject: [PATCH 09/17] handle the logs length in SetLogs --- x/evm/keeper/keeper.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index f6124275d7..b7d8d7bc2a 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -301,7 +301,12 @@ func (k Keeper) SetLogs(txHash common.Hash, logs []*ethtypes.Log) { len := len(logs) if len > 65535 { - //TODO: to log a warning, but still allows to store the logs? + k.Logger(k.Ctx()).Error( + "cannot store more than 65535 logs in a transaction", + "txhash", txHash.String(), + "log-length", len, + ) + return } else if len > 0 { s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) From 3ceb5367042265b4cef48a2a38afb01e2e615d29 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Thu, 26 Aug 2021 19:43:22 -0400 Subject: [PATCH 10/17] revise Get/Set logs with store.iterator --- x/evm/keeper/grpc_query_test.go | 4 +- x/evm/keeper/keeper.go | 122 ++++++++++---------------------- x/evm/keeper/statedb.go | 18 ++--- x/evm/types/key.go | 1 - 4 files changed, 45 insertions(+), 100 deletions(-) diff --git a/x/evm/keeper/grpc_query_test.go b/x/evm/keeper/grpc_query_test.go index edb0df2a34..7de674adb6 100644 --- a/x/evm/keeper/grpc_query_test.go +++ b/x/evm/keeper/grpc_query_test.go @@ -474,7 +474,7 @@ func (suite *KeeperTestSuite) TestQueryBlockLogs() { TxHash: ethcmn.BytesToHash([]byte("tx_hash_1")).String(), TxIndex: 1, BlockHash: ethcmn.BytesToHash([]byte("block_hash")).String(), - Index: 0, + Index: 1, Removed: false, }, { @@ -485,7 +485,7 @@ func (suite *KeeperTestSuite) TestQueryBlockLogs() { TxHash: ethcmn.BytesToHash([]byte("tx_hash_1")).String(), TxIndex: 1, BlockHash: ethcmn.BytesToHash([]byte("block_hash")).String(), - Index: 0, + Index: 2, Removed: false, }, }, diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index b7d8d7bc2a..0acf5124a5 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -229,126 +229,78 @@ func (k Keeper) ResetRefundTransient(ctx sdk.Context) { // GetAllTxLogs return all the transaction logs from the store. func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { store := ctx.KVStore(k.storeKey) - iterator := sdk.KVStorePrefixIterator(store, types.KeyPrefixTxLogCount) - defer iterator.Close() + iter := sdk.KVStorePrefixIterator(store, types.KeyPrefixLogs) + defer iter.Close() - txsLogs := []types.TransactionLogs{} - for ; iterator.Valid(); iterator.Next() { - txLogCount := types.BytesToUint16(iterator.Value()) - txHash := iterator.Key() - - var txlogs types.TransactionLogs - txlogs.Hash = common.Bytes2Hex(txHash) - - s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - for i := uint16(0); i < txLogCount; i++ { - var key []byte - key = append(key, txHash...) - key = append(key, types.Uint16ToBytes(i)...) - logbz := s.Get(key) - if logbz == nil { - // TODO: error handle? - continue - } - - var txLog types.Log - k.cdc.MustUnmarshal(logbz, &txLog) - txlogs.Logs = append(txlogs.Logs, &txLog) - } + var mapLogs = make(map[string][]*types.Log) + for ; iter.Valid(); iter.Next() { + var txLog types.Log + k.cdc.MustUnmarshal(iter.Value(), &txLog) - // add a new entry - txsLogs = append(txsLogs, txlogs) + txlogs := mapLogs[txLog.TxHash] + txlogs = append(txlogs, &txLog) + mapLogs[txLog.TxHash] = txlogs + } + + txsLogs := []types.TransactionLogs{} + for txHash, logs := range mapLogs { + txLogs := types.TransactionLogs{Hash: txHash, Logs: logs} + txsLogs = append(txsLogs, txLogs) } + return txsLogs } // GetLogs returns the current logs for a given transaction hash from the KVStore. // This function returns an empty, non-nil slice if no logs are found. func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { - store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) + store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - bz := store.Get(txHash.Bytes()) - if len(bz) == 0 { - return []*ethtypes.Log{} - } + var end = txHash.Bytes() + end = append(end, []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}...) - store = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - txLogCount := types.BytesToUint16(bz) + iter := store.Iterator(txHash.Bytes(), end) + defer iter.Close() logs := []*ethtypes.Log{} - for i := uint16(0); i < txLogCount; i++ { - var key []byte - key = append(key, txHash.Bytes()...) - key = append(key, types.Uint16ToBytes(i)...) - logbz := store.Get(key) - - if logbz == nil { - // TODO: how to handle this? - continue - } - + for ; iter.Valid(); iter.Next() { var log types.Log - k.cdc.MustUnmarshal(logbz, &log) - + k.cdc.MustUnmarshal(iter.Value(), &log) logs = append(logs, log.ToEthereum()) } + return logs } // SetLogs sets the logs for a transaction in the KVStore. func (k Keeper) SetLogs(txHash common.Hash, logs []*ethtypes.Log) { - var logStore prefix.Store - len := len(logs) - - if len > 65535 { - k.Logger(k.Ctx()).Error( - "cannot store more than 65535 logs in a transaction", - "txhash", txHash.String(), - "log-length", len, - ) - return - } else if len > 0 { - s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) - - s.Set(txHash.Bytes(), types.Uint16ToBytes(uint16(len))) - logStore = prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - } - - for index, log := range logs { - k.SetLog(txHash, log, logStore, uint16(index)) + store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + for _, log := range logs { + var key = txHash.Bytes() + key = append(key, sdk.Uint64ToBigEndian(uint64(log.Index))...) + k.SetLog(store, key, log) } } // SetLog sets the log for a transaction in the KVStore. -func (k Keeper) SetLog(txHash common.Hash, log *ethtypes.Log, s prefix.Store, index uint16) { +func (k Keeper) SetLog(s prefix.Store, key []byte, log *ethtypes.Log) { txIndexLog := types.NewLogFromEth(log) bz := k.cdc.MustMarshal(txIndexLog) - - var key []byte - key = append(key, txHash.Bytes()...) - key = append(key, types.Uint16ToBytes(index)...) s.Set(key, bz) } // DeleteLogs removes the logs from the KVStore. It is used during journal.Revert. func (k Keeper) DeleteTxLogs(ctx sdk.Context, txHash common.Hash) { - store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixTxLogCount) - - b := store.Get(txHash.Bytes()) - if len(b) == 0 { - return - } + store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) - txLogCount := types.BytesToUint16(b) - store.Delete(txHash.Bytes()) + var end = txHash.Bytes() + end = append(end, []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}...) - store = prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) - for i := uint16(0); i < txLogCount; i++ { - var key []byte - key = append(key, txHash.Bytes()...) - key = append(key, types.Uint16ToBytes(i)...) + iter := store.Iterator(txHash.Bytes(), end) + defer iter.Close() - store.Delete(key) + for ; iter.Valid(); iter.Next() { + store.Delete(iter.Key()) } } diff --git a/x/evm/keeper/statedb.go b/x/evm/keeper/statedb.go index 26ac0d96ff..27b1fc9e00 100644 --- a/x/evm/keeper/statedb.go +++ b/x/evm/keeper/statedb.go @@ -608,18 +608,12 @@ func (k *Keeper) AddLog(log *ethtypes.Log) { log.Index = uint(k.GetLogSizeTransient()) k.IncreaseLogSizeTransient() - s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixTxLogCount) - txLogCount := types.BytesToUint16(s.Get(log.TxHash.Bytes())) - if txLogCount == 65535 { - k.Logger(k.Ctx()).Error( - "failed to add log due to the log size limit", - "tx-hash-ethereum", log.TxHash.Hex(), - "log-index", int(log.Index), - ) - return - } - s.Set(log.TxHash.Bytes(), types.Uint16ToBytes(txLogCount+1)) - k.SetLog(log.TxHash, log, prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs), txLogCount) + s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + + var key = log.TxHash.Bytes() + key = append(key, sdk.Uint64ToBigEndian(uint64(log.Index))...) + + k.SetLog(s, key, log) k.Logger(k.Ctx()).Debug( "log added", diff --git a/x/evm/types/key.go b/x/evm/types/key.go index 29846c5fe7..577ed5f9c6 100644 --- a/x/evm/types/key.go +++ b/x/evm/types/key.go @@ -53,7 +53,6 @@ var ( KeyPrefixLogs = []byte{prefixLogs} KeyPrefixCode = []byte{prefixCode} KeyPrefixStorage = []byte{prefixStorage} - KeyPrefixTxLogCount = []byte{prefixTxLogCount} ) // Transient Store key prefixes From 02b59f844f9997bfa2db872e1d09bab858031d31 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Thu, 26 Aug 2021 19:48:19 -0400 Subject: [PATCH 11/17] remove Bytes<->Uint16 functions --- x/evm/types/utils.go | 26 -------------------------- x/evm/types/utils_test.go | 21 --------------------- 2 files changed, 47 deletions(-) diff --git a/x/evm/types/utils.go b/x/evm/types/utils.go index 43b101ad78..44ee600a7a 100644 --- a/x/evm/types/utils.go +++ b/x/evm/types/utils.go @@ -1,7 +1,6 @@ package types import ( - "encoding/binary" "fmt" "github.com/gogo/protobuf/proto" @@ -88,28 +87,3 @@ func BinSearch(lo uint64, hi uint64, executable func(uint64) (bool, *MsgEthereum } return hi, nil } - -// Uint16ToBytes transfer value to variable size []byte -func Uint16ToBytes(v uint16) []byte { - bz := []byte{0x00, 0x00} - binary.BigEndian.PutUint16(bz, v) - - if v < 256 { - return bz[1:] - } - - return bz -} - -// BytesToUint16 transfer []byte to uint16. -// Note: If the bytes length greater than two, the value lower than 256 will be returned. -func BytesToUint16(bz []byte) uint16 { - l := len(bz) - if l == 0 { - return 0 - } else if l == 1 { - return uint16(bz[0]) - } - - return binary.BigEndian.Uint16(bz[l-2:]) -} diff --git a/x/evm/types/utils_test.go b/x/evm/types/utils_test.go index dcaf83e749..b1fa74b95c 100644 --- a/x/evm/types/utils_test.go +++ b/x/evm/types/utils_test.go @@ -98,24 +98,3 @@ func TestBinSearch(t *testing.T) { require.Error(t, err) require.Equal(t, gas, uint64(0)) } - -func TestUint16ToBytes(t *testing.T) { - require.Equal(t, []byte{0x00}, evmtypes.Uint16ToBytes(0)) - require.Equal(t, []byte{0x01}, evmtypes.Uint16ToBytes(1)) - require.Equal(t, []byte{0xff}, evmtypes.Uint16ToBytes(255)) - require.Equal(t, []byte{0x01, 0x00}, evmtypes.Uint16ToBytes(256)) - require.Equal(t, []byte{0xff, 0xff}, evmtypes.Uint16ToBytes(65535)) -} - -func TestBytesToUint16(t *testing.T) { - require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{})) - require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{0x00})) - require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{0x00, 0x00})) - require.Equal(t, uint16(0), evmtypes.BytesToUint16([]byte{0x01, 0x00, 0x00})) - require.Equal(t, uint16(1), evmtypes.BytesToUint16([]byte{0x01})) - require.Equal(t, uint16(1), evmtypes.BytesToUint16([]byte{0x01, 0x00, 0x01})) - require.Equal(t, uint16(255), evmtypes.BytesToUint16([]byte{0xff})) - require.Equal(t, uint16(256), evmtypes.BytesToUint16([]byte{0x01, 0x00})) - require.Equal(t, uint16(65535), evmtypes.BytesToUint16([]byte{0xff, 0xff})) - require.Equal(t, uint16(65535), evmtypes.BytesToUint16([]byte{0x01, 0xff, 0xff})) -} From 1dc62ef2d9d6fd73b2a394bd4223e9bc595de595 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Thu, 26 Aug 2021 19:51:48 -0400 Subject: [PATCH 12/17] remove unused const --- x/evm/types/key.go | 1 - 1 file changed, 1 deletion(-) diff --git a/x/evm/types/key.go b/x/evm/types/key.go index 577ed5f9c6..02de5939cb 100644 --- a/x/evm/types/key.go +++ b/x/evm/types/key.go @@ -31,7 +31,6 @@ const ( prefixLogs prefixCode prefixStorage - prefixTxLogCount ) // prefix bytes for the EVM transient store From 389beda9f80b4fa793792936c01cb699bd20c727 Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 27 Aug 2021 10:09:01 -0400 Subject: [PATCH 13/17] simplify SetLog(s) --- x/evm/keeper/keeper.go | 14 +++++++++++--- x/evm/keeper/statedb.go | 8 +------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index 0acf5124a5..3c6177fda7 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -275,18 +275,26 @@ func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { // SetLogs sets the logs for a transaction in the KVStore. func (k Keeper) SetLogs(txHash common.Hash, logs []*ethtypes.Log) { store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + for _, log := range logs { var key = txHash.Bytes() key = append(key, sdk.Uint64ToBigEndian(uint64(log.Index))...) - k.SetLog(store, key, log) + txIndexLog := types.NewLogFromEth(log) + bz := k.cdc.MustMarshal(txIndexLog) + store.Set(key, bz) } } // SetLog sets the log for a transaction in the KVStore. -func (k Keeper) SetLog(s prefix.Store, key []byte, log *ethtypes.Log) { +func (k Keeper) SetLog(log *ethtypes.Log) { + store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + + var key = log.TxHash.Bytes() + key = append(key, sdk.Uint64ToBigEndian(uint64(log.Index))...) + txIndexLog := types.NewLogFromEth(log) bz := k.cdc.MustMarshal(txIndexLog) - s.Set(key, bz) + store.Set(key, bz) } // DeleteLogs removes the logs from the KVStore. It is used during journal.Revert. diff --git a/x/evm/keeper/statedb.go b/x/evm/keeper/statedb.go index 27b1fc9e00..a99e08dc41 100644 --- a/x/evm/keeper/statedb.go +++ b/x/evm/keeper/statedb.go @@ -607,13 +607,7 @@ func (k *Keeper) AddLog(log *ethtypes.Log) { log.Index = uint(k.GetLogSizeTransient()) k.IncreaseLogSizeTransient() - - s := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) - - var key = log.TxHash.Bytes() - key = append(key, sdk.Uint64ToBigEndian(uint64(log.Index))...) - - k.SetLog(s, key, log) + k.SetLog(log) k.Logger(k.Ctx()).Debug( "log added", From 78c63c8e58479a7dd59e6d4ab309204696f1c23e Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 27 Aug 2021 11:16:42 -0400 Subject: [PATCH 14/17] make logs return determinism --- x/evm/keeper/grpc_query.go | 12 ++++++++++-- x/evm/keeper/keeper.go | 14 ++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/x/evm/keeper/grpc_query.go b/x/evm/keeper/grpc_query.go index 5ae36d0e77..457b84a197 100644 --- a/x/evm/keeper/grpc_query.go +++ b/x/evm/keeper/grpc_query.go @@ -232,6 +232,8 @@ func (k Keeper) BlockLogs(c context.Context, req *types.QueryBlockLogsRequest) ( ctx := sdk.UnwrapSDKContext(c) store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) + + mapOrder := []string{} logs := make(map[string][]*types.Log) pageRes, err := query.FilteredPaginate(store, req.Pagination, func(_, value []byte, accumulate bool) (bool, error) { @@ -240,6 +242,10 @@ func (k Keeper) BlockLogs(c context.Context, req *types.QueryBlockLogsRequest) ( if txLog.BlockHash == req.Hash { if accumulate { + if len(logs[txLog.TxHash]) == 0 { + mapOrder = append(mapOrder, txLog.TxHash) + } + logs[txLog.TxHash] = append(logs[txLog.TxHash], &txLog) } return true, nil @@ -253,8 +259,10 @@ func (k Keeper) BlockLogs(c context.Context, req *types.QueryBlockLogsRequest) ( } var txsLogs = []types.TransactionLogs{} - for hash, log := range logs { - txsLogs = append(txsLogs, types.TransactionLogs{Hash: hash, Logs: log}) + for _, txHash := range mapOrder { + if len(logs[txHash]) > 0 { + txsLogs = append(txsLogs, types.TransactionLogs{Hash: txHash, Logs: logs[txHash]}) + } } return &types.QueryBlockLogsResponse{ diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index 3c6177fda7..a57c50a459 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -232,22 +232,28 @@ func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { iter := sdk.KVStorePrefixIterator(store, types.KeyPrefixLogs) defer iter.Close() + mapOrder := []string{} var mapLogs = make(map[string][]*types.Log) for ; iter.Valid(); iter.Next() { var txLog types.Log k.cdc.MustUnmarshal(iter.Value(), &txLog) txlogs := mapLogs[txLog.TxHash] + if len(txlogs) == 0 { + mapOrder = append(mapOrder, txLog.TxHash) + } + txlogs = append(txlogs, &txLog) mapLogs[txLog.TxHash] = txlogs } txsLogs := []types.TransactionLogs{} - for txHash, logs := range mapLogs { - txLogs := types.TransactionLogs{Hash: txHash, Logs: logs} - txsLogs = append(txsLogs, txLogs) + for _, txHash := range mapOrder { + if len(mapLogs[txHash]) > 0 { + txLogs := types.TransactionLogs{Hash: txHash, Logs: mapLogs[txHash]} + txsLogs = append(txsLogs, txLogs) + } } - return txsLogs } From 111a62b5f44de29029351027feb7a7f23a32e41b Mon Sep 17 00:00:00 2001 From: jayt106 Date: Fri, 27 Aug 2021 11:26:29 -0400 Subject: [PATCH 15/17] Add comments --- x/evm/keeper/keeper.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x/evm/keeper/keeper.go b/x/evm/keeper/keeper.go index a57c50a459..6af67b44ff 100644 --- a/x/evm/keeper/keeper.go +++ b/x/evm/keeper/keeper.go @@ -262,6 +262,8 @@ func (k Keeper) GetAllTxLogs(ctx sdk.Context) []types.TransactionLogs { func (k Keeper) GetTxLogs(txHash common.Hash) []*ethtypes.Log { store := prefix.NewStore(k.Ctx().KVStore(k.storeKey), types.KeyPrefixLogs) + // We store the logs with key equal to txHash.Bytes() | sdk.Uint64ToBigEndian(uint64(log.Index)), + // therefore, we set the end boundary(excluded) to txHash.Bytes() | uint64.Max -> []byte var end = txHash.Bytes() end = append(end, []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}...) @@ -307,6 +309,8 @@ func (k Keeper) SetLog(log *ethtypes.Log) { func (k Keeper) DeleteTxLogs(ctx sdk.Context, txHash common.Hash) { store := prefix.NewStore(ctx.KVStore(k.storeKey), types.KeyPrefixLogs) + // We store the logs with key equal to txHash.Bytes() | sdk.Uint64ToBigEndian(uint64(log.Index)), + // therefore, we set the end boundary(excluded) to txHash.Bytes() | uint64.Max -> []byte var end = txHash.Bytes() end = append(end, []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}...) From 670586b51d64c9f795faffe3ec8aa83c29013e5f Mon Sep 17 00:00:00 2001 From: jayt106 Date: Mon, 30 Aug 2021 10:38:07 -0400 Subject: [PATCH 16/17] update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e89558cc8..0b5faeedb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,9 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (encoding) [tharsis#478](https://github.com/tharsis/ethermint/pull/478) Register `Evidence` to amino codec. * (rpc) [tharsis#478](https://github.com/tharsis/ethermint/pull/481) Getting the node configuration when calling the `miner` rpc methods. +### Improvements +* (evm) [tharsis#461](https://github.com/tharsis/ethermint/issues/461) StateDB transaction log stores enhancement. + ## [v0.5.0] - 2021-08-20 ### State Machine Breaking From 39049521ae7169aa45eb7adf38129b4b4dd21b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Federico=20Kunze=20K=C3=BCllmer?= <31522760+fedekunze@users.noreply.github.com> Date: Mon, 30 Aug 2021 16:50:40 +0200 Subject: [PATCH 17/17] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b5faeedb1..3949982ec2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,7 +55,8 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (rpc) [tharsis#478](https://github.com/tharsis/ethermint/pull/481) Getting the node configuration when calling the `miner` rpc methods. ### Improvements -* (evm) [tharsis#461](https://github.com/tharsis/ethermint/issues/461) StateDB transaction log stores enhancement. + +* (evm) [tharsis#461](https://github.com/tharsis/ethermint/pull/461) Increase performance of `StateDB` transaction log storage (r/w). ## [v0.5.0] - 2021-08-20