Skip to content

Commit

Permalink
core: remove unnecessary fields in log (17016)
Browse files Browse the repository at this point in the history
  • Loading branch information
JukLee0ira committed Sep 19, 2024
1 parent a020259 commit 782bcdb
Show file tree
Hide file tree
Showing 12 changed files with 332 additions and 25 deletions.
8 changes: 7 additions & 1 deletion core/blockchain.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,13 @@ const (
triesInMemory = 128

// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
BlockChainVersion = 3
//
// During the process of upgrading the database version from 3 to 4,
// the following incompatible database changes were added.
// * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted
// * the `Bloom` field of receipt is deleted
// * the `BlockIndex` and `TxIndex` fields of txlookup are deleted
BlockChainVersion = 4

// Maximum length of chain to cache by block's number
blocksHashCacheLimit = 900
Expand Down
7 changes: 7 additions & 0 deletions core/rawdb/accessors_chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,13 @@ func WriteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rec
}
}

// DeleteReceipts removes all receipt data associated with a block hash.
func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
if err := db.Delete(blockReceiptsKey(number, hash)); err != nil {
log.Crit("Failed to delete block receipts", "err", err)
}
}

// storedReceiptRLP is the storage encoding of a receipt.
// Re-definition in core/types/receipt.go.
type storedReceiptRLP struct {
Expand Down
119 changes: 119 additions & 0 deletions core/rawdb/accessors_indexes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package rawdb

import (
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/rlp"
)

// ReadTxLookupEntry retrieves the positional metadata associated with a transaction
// hash to allow retrieving the transaction or receipt by hash.
func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) common.Hash {
data, _ := db.Get(txLookupKey(hash))
if len(data) == 0 {
return common.Hash{}
}
if len(data) == common.HashLength {
return common.BytesToHash(data)
}
// Probably it's legacy txlookup entry data, try to decode it.
var entry LegacyTxLookupEntry
if err := rlp.DecodeBytes(data, &entry); err != nil {
log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err)
return common.Hash{}
}
return entry.BlockHash
}

// WriteTxLookupEntries stores a positional metadata for every transaction from
// a block, enabling hash based transaction and receipt lookups.
func WriteTxLookupEntries(db ethdb.KeyValueWriter, block *types.Block) {
for _, tx := range block.Transactions() {
if err := db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()); err != nil {
log.Crit("Failed to store transaction lookup entry", "err", err)
}
}
}

// DeleteTxLookupEntry removes all transaction data associated with a hash.
func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) {
db.Delete(txLookupKey(hash))
}

// ReadTransaction retrieves a specific transaction from the database, along with
// its added positional metadata.
func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
blockHash := ReadTxLookupEntry(db, hash)
if blockHash == (common.Hash{}) {
return nil, common.Hash{}, 0, 0
}
blockNumber := ReadHeaderNumber(db, blockHash)
if blockNumber == nil {
return nil, common.Hash{}, 0, 0
}
body := ReadBody(db, blockHash, *blockNumber)
if body == nil {
log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash)
return nil, common.Hash{}, 0, 0
}
for txIndex, tx := range body.Transactions {
if tx.Hash() == hash {
return tx, blockHash, *blockNumber, uint64(txIndex)
}
}
log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
return nil, common.Hash{}, 0, 0
}

// // ReadReceipt retrieves a specific transaction receipt from the database, along with
// // its added positional metadata.
// func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Hash, uint64, uint64) {
// blockHash := ReadTxLookupEntry(db, hash)
// if blockHash == (common.Hash{}) {
// return nil, common.Hash{}, 0, 0
// }
// blockNumber := ReadHeaderNumber(db, blockHash)
// if blockNumber == nil {
// return nil, common.Hash{}, 0, 0
// }
// receipts := ReadReceipts(db, blockHash, *blockNumber)
// for receiptIndex, receipt := range receipts {
// if receipt.TxHash == hash {
// return receipt, blockHash, *blockNumber, uint64(receiptIndex)
// }
// }
// log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
// return nil, common.Hash{}, 0, 0
// }

// // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given
// // section and bit index from the.
// func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) {
// return db.Get(bloomBitsKey(bit, section, head))
// }

// // WriteBloomBits stores the compressed bloom bits vector belonging to the given
// // section and bit index.
// func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) {
// if err := db.Put(bloomBitsKey(bit, section, head), bits); err != nil {
// log.Crit("Failed to store bloom bits", "err", err)
// }
// }
90 changes: 90 additions & 0 deletions core/rawdb/accessors_indexes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package rawdb

import (
"math/big"
"testing"

"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/rlp"
)

// Tests that positional lookup metadata can be stored and retrieved.
func TestLookupStorage(t *testing.T) {
db := NewMemoryDatabase()

tx1 := types.NewTransaction(1, common.BytesToAddress([]byte{0x11}), big.NewInt(111), 1111, big.NewInt(11111), []byte{0x11, 0x11, 0x11})
tx2 := types.NewTransaction(2, common.BytesToAddress([]byte{0x22}), big.NewInt(222), 2222, big.NewInt(22222), []byte{0x22, 0x22, 0x22})
tx3 := types.NewTransaction(3, common.BytesToAddress([]byte{0x33}), big.NewInt(333), 3333, big.NewInt(33333), []byte{0x33, 0x33, 0x33})
txs := []*types.Transaction{tx1, tx2, tx3}

block := types.NewBlock(&types.Header{Number: big.NewInt(314)}, txs, nil, nil)

// Check that no transactions entries are in a pristine database
for i, tx := range txs {
if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
t.Fatalf("tx #%d [%x]: non existent transaction returned: %v", i, tx.Hash(), txn)
}
}
// Insert all the transactions into the database, and verify contents
WriteBlock(db, block)
WriteTxLookupEntries(db, block)

for i, tx := range txs {
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
} else {
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
}
if tx.Hash() != txn.Hash() {
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
}
}
}
// Delete the transactions and check purge
for i, tx := range txs {
DeleteTxLookupEntry(db, tx.Hash())
if txn, _, _, _ := ReadTransaction(db, tx.Hash()); txn != nil {
t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn)
}
}
// Insert legacy txlookup and verify the data retrieval
for index, tx := range block.Transactions() {
entry := LegacyTxLookupEntry{
BlockHash: block.Hash(),
BlockIndex: block.NumberU64(),
Index: uint64(index),
}
data, _ := rlp.EncodeToBytes(entry)
db.Put(txLookupKey(tx.Hash()), data)
}
for i, tx := range txs {
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
} else {
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
}
if tx.Hash() != txn.Hash() {
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
}
}
}
}
37 changes: 37 additions & 0 deletions core/rawdb/accessors_metadata.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package rawdb

import (
"github.com/XinFinOrg/XDPoSChain/ethdb"
"github.com/XinFinOrg/XDPoSChain/rlp"
)

// ReadDatabaseVersion retrieves the version number of the database.
func ReadDatabaseVersion(db ethdb.KeyValueReader) *uint64 {
var version uint64

enc, _ := db.Get(databaseVersionKey)
if len(enc) == 0 {
return nil
}
if err := rlp.DecodeBytes(enc, &version); err != nil {
return nil
}

return &version
}
20 changes: 20 additions & 0 deletions core/rawdb/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,21 @@ import (

// The fields below define the low level database schema prefixing.
var (
// databaseVersionKey tracks the current database version.
databaseVersionKey = []byte("DatabaseVersion")

// headBlockKey tracks the latest known full block's hash.
headBlockKey = []byte("LastBlock")

// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash
headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian)

blockBodyPrefix = []byte("b") // blockBodyPrefix + num (uint64 big endian) + hash -> block body
blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts

txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
)

const (
Expand All @@ -46,6 +53,14 @@ const (
freezerReceiptTable = "receipts"
)

// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary
// fields.
type LegacyTxLookupEntry struct {
BlockHash common.Hash
BlockIndex uint64
Index uint64
}

// encodeBlockNumber encodes a block number as big endian uint64
func encodeBlockNumber(number uint64) []byte {
enc := make([]byte, 8)
Expand Down Expand Up @@ -77,3 +92,8 @@ func blockBodyKey(number uint64, hash common.Hash) []byte {
func blockReceiptsKey(number uint64, hash common.Hash) []byte {
return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
}

// txLookupKey = txLookupPrefix + hash
func txLookupKey(hash common.Hash) []byte {
return append(txLookupPrefix, hash.Bytes()...)
}
41 changes: 24 additions & 17 deletions core/types/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ type rlpLog struct {
Data []byte
}

type rlpStorageLog struct {
// rlpStorageLog is the storage encoding of a log.
type rlpStorageLog rlpLog

// LegacyRlpStorageLog is the previous storage encoding of a log including some redundant fields.
type LegacyRlpStorageLog struct {
Address common.Address
Topics []common.Hash
Data []byte
Expand Down Expand Up @@ -106,31 +110,34 @@ type LogForStorage Log
// EncodeRLP implements rlp.Encoder.
func (l *LogForStorage) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, rlpStorageLog{
Address: l.Address,
Topics: l.Topics,
Data: l.Data,
BlockNumber: l.BlockNumber,
TxHash: l.TxHash,
TxIndex: l.TxIndex,
BlockHash: l.BlockHash,
Index: l.Index,
Address: l.Address,
Topics: l.Topics,
Data: l.Data,
})
}

// DecodeRLP implements rlp.Decoder.
//
// Note some redundant fields(e.g. block number, tx hash etc) will be assembled later.
func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
var dec rlpStorageLog
err := s.Decode(&dec)
if err == nil {
*l = LogForStorage{
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
BlockNumber: dec.BlockNumber,
TxHash: dec.TxHash,
TxIndex: dec.TxIndex,
BlockHash: dec.BlockHash,
Index: dec.Index,
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
}
} else {
// Try to decode log with previous definition.
var dec LegacyRlpStorageLog
err = s.Decode(&dec)
if err == nil {
*l = LogForStorage{
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
}
}
}
return err
Expand Down
Loading

0 comments on commit 782bcdb

Please sign in to comment.