RPC for staking txns and txns history queries (#2554)

* staking txn. look up by hash fix on api backend rawdb storage

* node explorer staking txn 'history' RPC layer support

* fix unit test

* add error log when explorer node db instance cannot be fetched

* revert unwanted merge changes during rebase

* use already encoded tx message fields for get staking txn rpc

* update explorer node storage service for staking txns

* use hex string for staking transaction data field

* revert transaction pool apiv1 changes

api/service/explorer/storage.go

@@ -8,6 +8,7 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/internal/utils"
+	staking "github.com/harmony-one/harmony/staking/types"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -83,25 +84,40 @@ func (storage *Storage) Dump(block *types.Block, height uint64) {
 	// Store txs
 	for _, tx := range block.Transactions() {
 		explorerTransaction := GetTransaction(tx, block)
-		storage.UpdateAddress(batch, explorerTransaction, tx)
+		storage.UpdateTxAddress(batch, explorerTransaction, tx)
+	}
+	// Store staking txns
+	for _, tx := range block.StakingTransactions() {
+		explorerTransaction := GetStakingTransaction(tx, block)
+		storage.UpdateStakingTxAddress(batch, explorerTransaction, tx)
 	}
 	if err := storage.db.Write(batch, nil); err != nil {
 		utils.Logger().Warn().Err(err).Msg("cannot write batch")
 	}
 }
 
-// UpdateAddress ...
-func (storage *Storage) UpdateAddress(batch *leveldb.Batch, explorerTransaction *Transaction, tx *types.Transaction) {
+// UpdateTxAddress ...
+func (storage *Storage) UpdateTxAddress(batch *leveldb.Batch, explorerTransaction *Transaction, tx *types.Transaction) {
+	explorerTransaction.Type = Received
+	if explorerTransaction.To != "" {
+		storage.UpdateTxAddressStorage(batch, explorerTransaction.To, explorerTransaction, tx)
+	}
+	explorerTransaction.Type = Sent
+	storage.UpdateTxAddressStorage(batch, explorerTransaction.From, explorerTransaction, tx)
+}
+
+// UpdateStakingTxAddress ...
+func (storage *Storage) UpdateStakingTxAddress(batch *leveldb.Batch, explorerTransaction *StakingTransaction, tx *staking.StakingTransaction) {
 	explorerTransaction.Type = Received
 	if explorerTransaction.To != "" {
-		storage.UpdateAddressStorage(batch, explorerTransaction.To, explorerTransaction, tx)
+		storage.UpdateStakingTxAddressStorage(batch, explorerTransaction.To, explorerTransaction, tx)
 	}
 	explorerTransaction.Type = Sent
-	storage.UpdateAddressStorage(batch, explorerTransaction.From, explorerTransaction, tx)
+	storage.UpdateStakingTxAddressStorage(batch, explorerTransaction.From, explorerTransaction, tx)
 }
 
-// UpdateAddressStorage updates specific addr Address.
-func (storage *Storage) UpdateAddressStorage(batch *leveldb.Batch, addr string, explorerTransaction *Transaction, tx *types.Transaction) {
+// UpdateTxAddressStorage updates specific addr tx Address.
+func (storage *Storage) UpdateTxAddressStorage(batch *leveldb.Batch, addr string, explorerTransaction *Transaction, tx *types.Transaction) {
 	var address Address
 	key := GetAddressKey(addr)
 	if data, err := storage.db.Get([]byte(key), nil); err == nil {
@@ -119,6 +135,25 @@ func (storage *Storage) UpdateAddressStorage(batch *leveldb.Batch, addr string,
 	}
 }
 
+// UpdateStakingTxAddressStorage updates specific addr staking tx Address.
+func (storage *Storage) UpdateStakingTxAddressStorage(batch *leveldb.Batch, addr string, explorerTransaction *StakingTransaction, tx *staking.StakingTransaction) {
+	var address Address
+	key := GetAddressKey(addr)
+	if data, err := storage.db.Get([]byte(key), nil); err == nil {
+		if err = rlp.DecodeBytes(data, &address); err != nil {
+			utils.Logger().Error().Err(err).Msg("Failed due to error")
+		}
+	}
+	address.ID = addr
+	address.StakingTXs = append(address.StakingTXs, explorerTransaction)
+	encoded, err := rlp.EncodeToBytes(address)
+	if err == nil {
+		batch.Put([]byte(key), encoded)
+	} else {
+		utils.Logger().Error().Err(err).Msg("cannot encode address")
+	}
+}
+
 // GetAddresses returns size of addresses from address with prefix.
 func (storage *Storage) GetAddresses(size int, prefix string) ([]string, error) {
 	db := storage.GetDB()

api/service/explorer/structs.go

@@ -5,9 +5,11 @@ import (
 	"math/big"
 	"strconv"
 
+	core2 "github.com/harmony-one/harmony/core"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/internal/common"
 	"github.com/harmony-one/harmony/internal/utils"
+	staking "github.com/harmony-one/harmony/staking/types"
 )
 
 /*
@@ -27,9 +29,10 @@ type Data struct {
 
 // Address ...
 type Address struct {
-	ID      string         `json:"id"`
-	Balance *big.Int       `json:"balance"`
-	TXs     []*Transaction `json:"txs"`
+	ID         string                `json:"id"`
+	Balance    *big.Int              `json:"balance"`
+	TXs        []*Transaction        `json:"txs"`
+	StakingTXs []*StakingTransaction `json:"staking_txs"`
 }
 
 // Transaction ...
@@ -79,3 +82,51 @@ func GetTransaction(tx *types.Transaction, addressBlock *types.Block) *Transacti
 		Type:      "",
 	}
 }
+
+// StakingTransaction ...
+type StakingTransaction struct {
+	ID        string   `json:"id"`
+	Timestamp string   `json:"timestamp"`
+	From      string   `json:"from"`
+	To        string   `json:"to"`
+	Value     *big.Int `json:"value"`
+	Bytes     string   `json:"bytes"`
+	Data      string   `json:"data"`
+	GasFee    *big.Int `json:"gasFee"`
+	FromShard uint32   `json:"fromShard"`
+	ToShard   uint32   `json:"toShard"`
+	Type      string   `json:"type"`
+}
+
+// GetStakingTransaction ...
+func GetStakingTransaction(tx *staking.StakingTransaction, addressBlock *types.Block) *StakingTransaction {
+	msg, err := core2.StakingToMessage(tx, addressBlock.Header().Number())
+	if err != nil {
+		utils.Logger().Error().Err(err).Msg("Error when parsing tx into message")
+	}
+	gasFee := big.NewInt(0)
+	gasFee = gasFee.Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas()))
+	to := ""
+	if msg.To() != nil {
+		if to, err = common.AddressToBech32(*msg.To()); err != nil {
+			return nil
+		}
+	}
+	from := ""
+	if from, err = common.AddressToBech32(msg.From()); err != nil {
+		return nil
+	}
+	return &StakingTransaction{
+		ID:        tx.Hash().Hex(),
+		Timestamp: strconv.Itoa(int(addressBlock.Time().Int64() * 1000)),
+		From:      from,
+		To:        to,
+		Value:     msg.Value(),
+		Bytes:     strconv.Itoa(int(tx.Size())),
+		Data:      hex.EncodeToString(msg.Data()),
+		GasFee:    gasFee,
+		FromShard: tx.ShardID(),
+		ToShard:   0,
+		Type:      string(msg.Type()),
+	}
+}

block/factory/factory.go

@@ -30,7 +30,7 @@ func NewFactory(chainConfig *params.ChainConfig) Factory {
 func (f *factory) NewHeader(epoch *big.Int) *block.Header {
 	var impl blockif.Header
 	switch {
-	case f.chainConfig.IsPreStaking(epoch):
+	case f.chainConfig.IsPreStaking(epoch) || f.chainConfig.IsStaking(epoch):
 		impl = v3.NewHeader()
 	case f.chainConfig.IsCrossLink(epoch):
 		impl = v2.NewHeader()

core/rawdb/accessors_indexes.go

@@ -45,33 +45,34 @@ func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64
 // WriteTxLookupEntries stores a positional metadata for every transaction from
 // a block, enabling hash based transaction and receipt lookups.
 func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) {
-	for i, tx := range block.Transactions() {
+	// TODO: remove this hack with Tx and StakingTx structure unitification later
+	f := func(i int, tx *types.Transaction, stx *staking.StakingTransaction) {
+		isStaking := (stx != nil && tx == nil)
 		entry := TxLookupEntry{
 			BlockHash:  block.Hash(),
 			BlockIndex: block.NumberU64(),
 			Index:      uint64(i),
 		}
 		data, err := rlp.EncodeToBytes(entry)
 		if err != nil {
-			utils.Logger().Error().Err(err).Msg("Failed to encode transaction lookup entry")
+			utils.Logger().Error().Err(err).Bool("isStaking", isStaking).Msg("Failed to encode transaction lookup entry")
+		}
+
+		var putErr error
+		if isStaking {
+			putErr = db.Put(txLookupKey(stx.Hash()), data)
+		} else {
+			putErr = db.Put(txLookupKey(tx.Hash()), data)
 		}
-		if err := db.Put(txLookupKey(tx.Hash()), data); err != nil {
-			utils.Logger().Error().Err(err).Msg("Failed to store transaction lookup entry")
+		if putErr != nil {
+			utils.Logger().Error().Err(err).Bool("isStaking", isStaking).Msg("Failed to store transaction lookup entry")
 		}
 	}
+	for i, tx := range block.Transactions() {
+		f(i, tx, nil)
+	}
 	for i, tx := range block.StakingTransactions() {
-		entry := TxLookupEntry{
-			BlockHash:  block.Hash(),
-			BlockIndex: block.NumberU64(),
-			Index:      uint64(i),
-		}
-		data, err := rlp.EncodeToBytes(entry)
-		if err != nil {
-			utils.Logger().Error().Err(err).Msg("Failed to encode staking transaction lookup entry")
-		}
-		if err := db.Put(txLookupKey(tx.Hash()), data); err != nil {
-			utils.Logger().Error().Err(err).Msg("Failed to store staking transaction lookup entry")
-		}
+		f(i, nil, tx)
 	}
 }
 
@@ -110,6 +111,8 @@ func ReadTransaction(db DatabaseReader, hash common.Hash) (*types.Transaction, c
 
 // ReadStakingTransaction retrieves a specific staking transaction from the database, along with
 // its added positional metadata.
+// TODO remove this duplicate function that is inevitable at the moment until the optimization on staking txn with
+// unification of txn vs staking txn data structure.
 func ReadStakingTransaction(db DatabaseReader, hash common.Hash) (*staking.StakingTransaction, common.Hash, uint64, uint64) {
 	blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash)
 	if blockHash == (common.Hash{}) {
@@ -143,6 +146,7 @@ func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Ha
 	if blockHash == (common.Hash{}) {
 		return nil, common.Hash{}, 0, 0
 	}
+
 	receipts := ReadReceipts(db, blockHash, blockNumber)
 	if len(receipts) <= int(receiptIndex) {
 		utils.Logger().Error().

core/rawdb/accessors_indexes_test.go

@@ -47,14 +47,29 @@ func TestLookupStorage(t *testing.T) {
 	tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), 0, big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
 	txs := []*types.Transaction{tx1, tx2, tx3}
 
-	block := types.NewBlock(blockfactory.NewTestHeader().With().Number(big.NewInt(314)).Header(), txs, types.Receipts{&types.Receipt{}, &types.Receipt{}, &types.Receipt{}}, nil, nil, nil)
+	stx := sampleCreateValidatorStakingTxn()
+	stxs := []*staking.StakingTransaction{stx}
+
+	receipts := types.Receipts{
+		&types.Receipt{},
+		&types.Receipt{},
+		&types.Receipt{},
+		&types.Receipt{},
+	}
+
+	block := types.NewBlock(blockfactory.NewTestHeader().With().Number(big.NewInt(314)).Header(), txs, receipts, nil, nil, stxs)
 
 	// Check that no transactions entries are in a pristine database
 	for i, tx := range txs {
 		if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
 			t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn)
 		}
 	}
+	for i, stx := range stxs {
+		if stxn, _, _, _ := ReadStakingTransaction(db, stx.Hash()); stxn != nil {
+			t.Fatalf("stx #%d [%x]: non existent staking transaction returned: %v", i, stxn.Hash(), stxn)
+		}
+	}
 	// Insert all the transactions into the database, and verify contents
 	WriteBlock(db, block)
 	WriteTxLookupEntries(db, block)
@@ -71,19 +86,56 @@ func TestLookupStorage(t *testing.T) {
 			}
 		}
 	}
+	for i, stx := range stxs {
+		if txn, hash, number, index := ReadStakingTransaction(db, stx.Hash()); txn == nil {
+			t.Fatalf("tx #%d [%x]: staking transaction not found", i, stx.Hash())
+		} else {
+			if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
+				t.Fatalf("stx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, stx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
+			}
+			if stx.Hash() != txn.Hash() {
+				t.Fatalf("stx #%d [%x]: staking transaction mismatch: have %v, want %v", i, stx.Hash(), txn, stx)
+			}
+		}
+	}
 	// Delete the transactions and check purge
 	for i, tx := range txs {
 		DeleteTxLookupEntry(db, tx.Hash())
 		if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
 			t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn)
 		}
 	}
+	for i, tx := range txs {
+		DeleteTxLookupEntry(db, tx.Hash())
+		if stxn, _, _, _ := ReadStakingTransaction(db, tx.Hash()); stxn != nil {
+			t.Fatalf("stx #%d [%x]: deleted staking transaction returned: %v", i, stx.Hash(), stxn)
+		}
+	}
 }
 
 // Test that staking tx hash does not find a plain tx hash (and visa versa) within the same block
 func TestMixedLookupStorage(t *testing.T) {
 	db := ethdb.NewMemDatabase()
 	tx := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), 0, big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
+	stx := sampleCreateValidatorStakingTxn()
+
+	txs := []*types.Transaction{tx}
+	stxs := []*staking.StakingTransaction{stx}
+	header := blockfactory.NewTestHeader().With().Number(big.NewInt(314)).Header()
+	block := types.NewBlock(header, txs, types.Receipts{&types.Receipt{}, &types.Receipt{}}, nil, nil, stxs)
+
+	WriteBlock(db, block)
+	WriteTxLookupEntries(db, block)
+
+	if recTx, _, _, _ := ReadStakingTransaction(db, tx.Hash()); recTx != nil {
+		t.Fatal("got staking transactions with plain tx hash")
+	}
+	if recTx, _, _, _ := ReadTransaction(db, stx.Hash()); recTx != nil {
+		t.Fatal("got plain transactions with staking tx hash")
+	}
+}
+
+func sampleCreateValidatorStakingTxn() *staking.StakingTransaction {
 	key, _ := crypto.GenerateKey()
 	stakePayloadMaker := func() (staking.Directive, interface{}) {
 		p := &bls.PublicKey{}
@@ -123,17 +175,5 @@ func TestMixedLookupStorage(t *testing.T) {
 		}
 	}
 	stx, _ := staking.NewStakingTransaction(0, 1e10, big.NewInt(10000), stakePayloadMaker)
-	txs := []*types.Transaction{tx}
-	stxs := []*staking.StakingTransaction{stx}
-	header := blockfactory.NewTestHeader().With().Number(big.NewInt(314)).Header()
-	block := types.NewBlock(header, txs, types.Receipts{&types.Receipt{}, &types.Receipt{}}, nil, nil, stxs)
-	WriteBlock(db, block)
-	WriteTxLookupEntries(db, block)
-
-	if recTx, _, _, _ := ReadStakingTransaction(db, tx.Hash()); recTx != nil {
-		t.Fatal("got staking transactions with plain tx hash")
-	}
-	if recTx, _, _, _ := ReadTransaction(db, stx.Hash()); recTx != nil {
-		t.Fatal("got plain transactions with staking tx hash")
-	}
+	return stx
 }

hmy/api_backend.go

@@ -239,6 +239,12 @@ func (b *APIBackend) GetTransactionsHistory(address, txType, order string) ([]co
 	return hashes, err
 }
 
+// GetStakingTransactionsHistory returns list of staking transactions hashes of address.
+func (b *APIBackend) GetStakingTransactionsHistory(address, txType, order string) ([]common.Hash, error) {
+	hashes, err := b.hmy.nodeAPI.GetStakingTransactionsHistory(address, txType, order)
+	return hashes, err
+}
+
 // NetVersion returns net version
 func (b *APIBackend) NetVersion() uint64 {
 	return b.hmy.NetVersion()
@@ -281,18 +287,26 @@ func (b *APIBackend) GetValidators(epoch *big.Int) (*shard.Committee, error) {
 }
 
 // ResendCx retrieve blockHash from txID and add blockHash to CxPool for resending
+// Note that cross shard txn is only for regular txns, not for staking txns, so the input txn hash
+// is expected to be regular txn hash
 func (b *APIBackend) ResendCx(ctx context.Context, txID common.Hash) (uint64, bool) {
 	blockHash, blockNum, index := b.hmy.BlockChain().ReadTxLookupEntry(txID)
+	if blockHash == (common.Hash{}) {
+		return 0, false
+	}
+
 	blk := b.hmy.BlockChain().GetBlockByHash(blockHash)
 	if blk == nil {
 		return 0, false
 	}
+
 	txs := blk.Transactions()
 	// a valid index is from 0 to len-1
 	if int(index) > len(txs)-1 {
 		return 0, false
 	}
 	tx := txs[int(index)]
+
 	// check whether it is a valid cross shard tx
 	if tx.ShardID() == tx.ToShardID() || blk.Header().ShardID() != tx.ShardID() {
 		return 0, false

hmy/backend.go

@@ -50,6 +50,7 @@ type NodeAPI interface {
 	GetBalanceOfAddress(address common.Address) (*big.Int, error)
 	GetNonceOfAddress(address common.Address) uint64
 	GetTransactionsHistory(address, txType, order string) ([]common.Hash, error)
+	GetStakingTransactionsHistory(address, txType, order string) ([]common.Hash, error)
 	IsCurrentlyLeader() bool
 	ErroredStakingTransactionSink() []staking.RPCTransactionError
 	ErroredTransactionSink() []types.RPCTransactionError