Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

trie, les, tests, core: implement trie tracer #24403

Merged
merged 1 commit into from
Mar 31, 2022
Merged

trie, les, tests, core: implement trie tracer #24403

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 9 additions & 4 deletions core/types/hashing_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
Expand All @@ -38,7 +39,8 @@ func TestDeriveSha(t *testing.T) {
t.Fatal(err)
}
for len(txs) < 1000 {
exp := types.DeriveSha(txs, new(trie.Trie))
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp := types.DeriveSha(txs, tr)
Comment on lines +42 to +43
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why did you make these changes? Old code seems to work fine, without instantiating new databases

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The original initialization new(trie.Trie) will create the trie structure but leave all pointers as nil.
Right now it's db and tracer. Although in this case both of them are not used, but it's still very dangerous to leave pointers as nil.

I did a lots of this kind changes to ensure all the created trie instances are initialized properly.

got := types.DeriveSha(txs, trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
Expand Down Expand Up @@ -85,7 +87,8 @@ func BenchmarkDeriveSha200(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
exp = types.DeriveSha(txs, new(trie.Trie))
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp = types.DeriveSha(txs, tr)
}
})

Expand All @@ -106,7 +109,8 @@ func TestFuzzDeriveSha(t *testing.T) {
rndSeed := mrand.Int()
for i := 0; i < 10; i++ {
seed := rndSeed + i
exp := types.DeriveSha(newDummy(i), new(trie.Trie))
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp := types.DeriveSha(newDummy(i), tr)
got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed))
Expand Down Expand Up @@ -134,7 +138,8 @@ func TestDerivableList(t *testing.T) {
},
}
for i, tc := range tcs[1:] {
exp := types.DeriveSha(flatList(tc), new(trie.Trie))
tr, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
exp := types.DeriveSha(flatList(tc), tr)
got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp)
Expand Down
2 changes: 1 addition & 1 deletion les/server_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -374,7 +374,7 @@ func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccou
return acc, nil
}

// getHelperTrie returns the post-processed trie root for the given trie ID and section index
// GetHelperTrie returns the post-processed trie root for the given trie ID and section index
func (h *serverHandler) GetHelperTrie(typ uint, index uint64) *trie.Trie {
var (
root common.Hash
Expand Down
4 changes: 2 additions & 2 deletions tests/fuzzers/rangeproof/rangeproof-fuzzer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"sort"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/trie"
)
Expand Down Expand Up @@ -61,8 +62,7 @@ func (f *fuzzer) readInt() uint64 {
}

func (f *fuzzer) randomTrie(n int) (*trie.Trie, map[string]*kv) {

trie := new(trie.Trie)
trie, _ := trie.New(common.Hash{}, trie.NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
size := f.readInt()
// Fill it with some fluff
Expand Down
2 changes: 1 addition & 1 deletion trie/committer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ func (c *committer) commit(n node, db *Database) (node, int, error) {
if hash != nil && !dirty {
return hash, 0, nil
}
// Commit children, then parent, and remove remove the dirty flag.
// Commit children, then parent, and remove the dirty flag.
switch cn := n.(type) {
case *shortNode:
// Commit child
Expand Down
3 changes: 2 additions & 1 deletion trie/iterator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"testing"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
Expand Down Expand Up @@ -296,7 +297,7 @@ func TestUnionIterator(t *testing.T) {
}

func TestIteratorNoDups(t *testing.T) {
var tr Trie
tr, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
for _, val := range testdata1 {
tr.Update([]byte(val.k), []byte(val.v))
}
Expand Down
3 changes: 1 addition & 2 deletions trie/proof.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/log"
)

Expand Down Expand Up @@ -552,7 +551,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key
}
// Rebuild the trie with the leaf stream, the shape of trie
// should be same with the original one.
tr := &Trie{root: root, db: NewDatabase(memorydb.New())}
tr := newWithRootNode(root)
if empty {
tr.root = nil
}
Expand Down
21 changes: 11 additions & 10 deletions trie/proof_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"time"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
)
Expand Down Expand Up @@ -79,7 +80,7 @@ func TestProof(t *testing.T) {
}

func TestOneElementProof(t *testing.T) {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "k", "v")
for i, prover := range makeProvers(trie) {
proof := prover([]byte("k"))
Expand Down Expand Up @@ -130,7 +131,7 @@ func TestBadProof(t *testing.T) {
// Tests that missing keys can also be proven. The test explicitly uses a single
// entry trie and checks for missing keys both before and after the single entry.
func TestMissingKeyProof(t *testing.T) {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
updateString(trie, "k", "v")

for i, key := range []string{"a", "j", "l", "z"} {
Expand Down Expand Up @@ -386,7 +387,7 @@ func TestOneElementRangeProof(t *testing.T) {
}

// Test the mini trie with only a single element.
tinyTrie := new(Trie)
tinyTrie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
entry := &kv{randBytes(32), randBytes(20), false}
tinyTrie.Update(entry.k, entry.v)

Expand Down Expand Up @@ -458,7 +459,7 @@ func TestAllElementsProof(t *testing.T) {
// TestSingleSideRangeProof tests the range starts from zero.
func TestSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
Expand Down Expand Up @@ -493,7 +494,7 @@ func TestSingleSideRangeProof(t *testing.T) {
// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff.
func TestReverseSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
Expand Down Expand Up @@ -600,7 +601,7 @@ func TestBadRangeProof(t *testing.T) {
// TestGappedRangeProof focuses on the small trie with embedded nodes.
// If the gapped node is embedded in the trie, it should be detected too.
func TestGappedRangeProof(t *testing.T) {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
var entries []*kv // Sorted entries
for i := byte(0); i < 10; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
Expand Down Expand Up @@ -674,7 +675,7 @@ func TestSameSideProofs(t *testing.T) {
}

func TestHasRightElement(t *testing.T) {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
var entries entrySlice
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
Expand Down Expand Up @@ -1027,7 +1028,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) {
}

func randomTrie(n int) (*Trie, map[string]*kv) {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
for i := byte(0); i < 100; i++ {
value := &kv{common.LeftPadBytes([]byte{i}, 32), []byte{i}, false}
Expand All @@ -1052,7 +1053,7 @@ func randBytes(n int) []byte {
}

func nonRandomTrie(n int) (*Trie, map[string]*kv) {
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
vals := make(map[string]*kv)
max := uint64(0xffffffffffffffff)
for i := uint64(0); i < uint64(n); i++ {
Expand All @@ -1077,7 +1078,7 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) {
common.Hex2Bytes("02"),
common.Hex2Bytes("03"),
}
trie := new(Trie)
trie, _ := New(common.Hash{}, NewDatabase(rawdb.NewMemoryDatabase()))
for i, key := range keys {
trie.Update(key, vals[i])
}
Expand Down
8 changes: 5 additions & 3 deletions trie/secure_trie.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) {
return t.trie.TryGetNode(path)
}

// TryUpdate account will abstract the write of an account to the
// TryUpdateAccount account will abstract the write of an account to the
// secure trie.
func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error {
hk := t.hashKey(key)
Expand Down Expand Up @@ -185,8 +185,10 @@ func (t *SecureTrie) Hash() common.Hash {

// Copy returns a copy of SecureTrie.
func (t *SecureTrie) Copy() *SecureTrie {
cpy := *t
return &cpy
return &SecureTrie{
trie: *t.trie.Copy(),
secKeyCache: t.secKeyCache,
}
}

// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration
Expand Down
3 changes: 1 addition & 2 deletions trie/secure_trie_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,7 @@ func TestSecureTrieConcurrency(t *testing.T) {
threads := runtime.NumCPU()
tries := make([]*SecureTrie, threads)
for i := 0; i < threads; i++ {
cpy := *trie
tries[i] = &cpy
tries[i] = trie.Copy()
}
// Start a batch of goroutines interactng with the trie
pend := new(sync.WaitGroup)
Expand Down
58 changes: 56 additions & 2 deletions trie/trie.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"sync"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
Expand Down Expand Up @@ -62,17 +63,32 @@ type LeafCallback func(paths [][]byte, hexpath []byte, leaf []byte, parent commo
type Trie struct {
db *Database
root node
// Keep track of the number leafs which have been inserted since the last

// Keep track of the number leaves which have been inserted since the last
// hashing operation. This number will not directly map to the number of
// actually unhashed nodes
unhashed int

// tracer is the state diff tracer can be used to track newly added/deleted
// trie node. It will be reset after each commit operation.
tracer *tracer
}

// newFlag returns the cache flag value for a newly created node.
func (t *Trie) newFlag() nodeFlag {
return nodeFlag{dirty: true}
}

// Copy returns a copy of Trie.
func (t *Trie) Copy() *Trie {
return &Trie{
db: t.db,
root: t.root,
unhashed: t.unhashed,
tracer: t.tracer.copy(),
}
}

// New creates a trie with an existing root node from db.
//
// If root is the zero hash or the sha3 hash of an empty string, the
Expand All @@ -85,6 +101,7 @@ func New(root common.Hash, db *Database) (*Trie, error) {
}
trie := &Trie{
db: db,
//tracer: newTracer(),
}
if root != (common.Hash{}) && root != emptyRoot {
rootnode, err := trie.resolveHash(root[:], nil)
Expand All @@ -96,6 +113,16 @@ func New(root common.Hash, db *Database) (*Trie, error) {
return trie, nil
}

// newWithRootNode initializes the trie with the given root node.
// It's only used by range prover.
func newWithRootNode(root node) *Trie {
return &Trie{
root: root,
//tracer: newTracer(),
db: NewDatabase(rawdb.NewMemoryDatabase()),
}
}

// NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at
// the key after the given start key.
func (t *Trie) NodeIterator(start []byte) NodeIterator {
Expand Down Expand Up @@ -317,7 +344,12 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
if matchlen == 0 {
return true, branch, nil
}
// Otherwise, replace it with a short node leading up to the branch.
// New branch node is created as a child of the original short node.
// Track the newly inserted node in the tracer. The node identifier
// passed is the path from the root node.
t.tracer.onInsert(append(prefix, key[:matchlen]...))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how does this not panic if no tracer is used?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's in tracer. If the tracer is nil, all the operations are ignored silently.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought that this might be costly -- even if the tracer is nil, there might be a function call + allocation for the data passed. However, I didn't notice anything special when running some benchmarks.


// Replace it with a short node leading up to the branch.
return true, &shortNode{key[:matchlen], branch, t.newFlag()}, nil

case *fullNode:
Expand All @@ -331,6 +363,11 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
return true, n, nil

case nil:
// New short node is created and track it in the tracer. The node identifier
// passed is the path from the root node. Note the valueNode won't be tracked
// since it's always embedded in its parent.
t.tracer.onInsert(prefix)

return true, &shortNode{key, value, t.newFlag()}, nil

case hashNode:
Expand Down Expand Up @@ -383,6 +420,11 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
return false, n, nil // don't replace n on mismatch
}
if matchlen == len(key) {
// The matched short node is deleted entirely and track
// it in the deletion set. The same the valueNode doesn't
// need to be tracked at all since it's always embedded.
t.tracer.onDelete(prefix)

return true, nil, nil // remove n entirely for whole matches
}
// The key is longer than n.Key. Remove the remaining suffix
Expand All @@ -395,6 +437,10 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
}
switch child := child.(type) {
case *shortNode:
// The child shortNode is merged into its parent, track
// is deleted as well.
t.tracer.onDelete(append(prefix, n.Key...))

// Deleting from the subtrie reduced it to another
// short node. Merge the nodes to avoid creating a
// shortNode{..., shortNode{...}}. Use concat (which
Expand Down Expand Up @@ -456,6 +502,11 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
return false, nil, err
}
if cnode, ok := cnode.(*shortNode); ok {
// Replace the entire full node with the short node.
// Mark the original short node as deleted since the
// value is embedded into the parent now.
t.tracer.onDelete(append(prefix, byte(pos)))

k := append([]byte{byte(pos)}, cnode.Key...)
return true, &shortNode{k, cnode.Val, t.newFlag()}, nil
}
Expand Down Expand Up @@ -537,6 +588,8 @@ func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) {
if t.db == nil {
panic("commit called on trie with nil database")
}
defer t.tracer.reset()

if t.root == nil {
return emptyRoot, 0, nil
}
Expand Down Expand Up @@ -595,4 +648,5 @@ func (t *Trie) hashRoot() (node, node, error) {
func (t *Trie) Reset() {
t.root = nil
t.unhashed = 0
t.tracer.reset()
}
Loading