Remove old stacktrie

cmd/geth/chaincmd.go

@@ -19,6 +19,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -201,6 +202,22 @@ Use "ethereum dump 0" to dump the genesis block.`,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 	}
+	generateTrieCommand = cli.Command{
+		Action:    utils.MigrateFlags(snapToHash),
+		Name:      "snaphash",
+		Usage:     "Calculate the trie root hash from the snapshot db",
+		ArgsUsage: " ",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.AncientFlag,
+			utils.CacheFlag,
+			utils.TestnetFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.SyncModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
+	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -576,6 +593,40 @@ func inspect(ctx *cli.Context) error {
 	return rawdb.InspectDatabase(chainDb)
 }
 
+func snapToHash(ctx *cli.Context) error {
+	node, _ := makeConfigNode(ctx)
+	chain, chainDb := utils.MakeChain(ctx, node)
+
+	defer func() {
+		node.Close()
+		chain.Stop()
+		chainDb.Close()
+	}()
+
+	snapTree := chain.Snapshot()
+	if snapTree == nil {
+		return fmt.Errorf("No snapshot tree available")
+	}
+	block := chain.CurrentBlock()
+	if block == nil {
+		return fmt.Errorf("no blocks present")
+	}
+	root := block.Root()
+	it, err := snapTree.AccountIterator(root, common.Hash{})
+	if err != nil {
+		return fmt.Errorf("Could not create iterator for root %x: %v", root, err)
+	}
+	generatedRoot := snapshot.GenerateTrieRoot(it)
+	if err := it.Error(); err != nil {
+		fmt.Printf("Iterator error: %v\n", it.Error())
+	}
+	if root != generatedRoot {
+		return fmt.Errorf("Wrong hash generated, expected %x, got %x", root, generatedRoot[:])
+	}
+	log.Info("Generation done", "root", generatedRoot)
+	return nil
+}
+
 // hashish returns true for strings that look like hashes.
 func hashish(x string) bool {
 	_, err := strconv.Atoi(x)

cmd/geth/main.go

@@ -210,6 +210,7 @@ func init() {
 		dumpCommand,
 		dumpGenesisCommand,
 		inspectCommand,
+		generateTrieCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,

cmd/utils/flags.go

@@ -1740,10 +1740,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 		TrieDirtyDisabled:   ctx.GlobalString(GCModeFlag.Name) == "archive",
 		TrieTimeLimit:       eth.DefaultConfig.TrieTimeout,
 		SnapshotLimit:       eth.DefaultConfig.SnapshotCache,
+		SnapshotWait:        false,
 	}
 	if !ctx.GlobalIsSet(SnapshotFlag.Name) {
 		cache.SnapshotLimit = 0 // Disabled
 	}
+	// Cover your eyes, this is a hack
+	if ctx.Command.Name == "snaphash" {
+		cache.SnapshotLimit = eth.DefaultConfig.SnapshotCache
+		cache.SnapshotWait = true
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
 		cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
 	}

core/blockchain.go

@@ -313,6 +313,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	return bc, nil
 }
 
+func (bc *BlockChain) Snapshot() *snapshot.Tree {
+	return bc.snaps
+}
+
 func (bc *BlockChain) getProcInterrupt() bool {
 	return atomic.LoadInt32(&bc.procInterrupt) == 1
 }

core/state/snapshot/account.go

@@ -21,7 +21,9 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"golang.org/x/crypto/sha3"
 )
 
 // Account is a slim version of a state.Account, where the root and code hash
@@ -52,3 +54,80 @@ func AccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byt
 	}
 	return data
 }
+
+func SlimToFull(data []byte) []byte {
+	acc := &Account{}
+	rlp.DecodeBytes(data, acc)
+	if len(acc.Root) == 0 {
+		acc.Root = emptyRoot[:]
+	}
+	if len(acc.CodeHash) == 0 {
+		acc.CodeHash = emptyCode[:]
+	}
+	fullData, err := rlp.EncodeToBytes(acc)
+	if err != nil {
+		panic(err)
+	}
+	return fullData
+}
+
+// conversionAccount is used for converting between full and slim format. When
+// doing this, we can consider 'balance' as a byte array, as it has already
+// been converted from big.Int into an rlp-byteslice.
+type conversionAccount struct {
+	Nonce    uint64
+	Balance  []byte
+	Root     []byte
+	CodeHash []byte
+}
+
+type converter struct {
+	tmpAcc *conversionAccount
+	sha3   crypto.KeccakState
+	stream rlp.Stream
+}
+
+func newConverter() *converter {
+	return &converter{
+		tmpAcc: &conversionAccount{},
+		sha3:   sha3.NewLegacyKeccak256().(crypto.KeccakState),
+	}
+}
+
+func (c *converter) SlimToHash(data []byte) common.Hash {
+	var (
+		result common.Hash
+		tmp    = c.tmpAcc
+		sha3   = c.sha3
+	)
+	c.stream.Reset(bytes.NewReader(data), 0)
+	c.stream.Decode(c.tmpAcc)
+	if len(tmp.Root) == 0 {
+		tmp.Root = emptyRoot[:]
+	}
+	if len(tmp.CodeHash) == 0 {
+		tmp.CodeHash = emptyCode[:]
+	}
+	sha3.Reset()
+	_ = rlp.Encode(sha3, tmp)
+	sha3.Read(result[:])
+	return result
+}
+
+// SlimToHash produces a hash of a main account trie, where the input is the
+// 'slim' version
+func SlimToHash(data []byte, sha3 crypto.KeccakState) common.Hash {
+	tmp := &conversionAccount{}
+	var result common.Hash
+	rlp.DecodeBytes(data, tmp)
+	if len(tmp.Root) == 0 {
+		tmp.Root = emptyRoot[:]
+	}
+	if len(tmp.CodeHash) == 0 {
+		tmp.CodeHash = emptyCode[:]
+	}
+	sha3.Reset()
+	_ = rlp.Encode(sha3, tmp)
+	sha3.Read(result[:])
+	return result
+}

core/state/snapshot/hextrie_generator.go

@@ -0,0 +1,177 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package snapshot
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/trie"
+)
+
+type leaf struct {
+	key   common.Hash
+	value []byte
+}
+
+type trieGeneratorFn func(in chan (leaf), out chan (common.Hash))
+
+// GenerateTrieRoot takes an account iterator and reproduces the root hash.
+func GenerateTrieRoot(it AccountIterator) common.Hash {
+	//return generateTrieRoot(it, StackGenerate)
+	return generateTrieRoot(it, ReStackGenerate)
+}
+
+func CrosscheckTriehasher(it AccountIterator, begin, end int) bool {
+	return verifyHasher(it, ReStackGenerate, begin, end)
+}
+
+func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Hash {
+	var (
+		in  = make(chan leaf)        // chan to pass leaves
+		out = make(chan common.Hash) // chan to collect result
+		wg  sync.WaitGroup
+	)
+	wg.Add(1)
+	go func() {
+		generatorFn(in, out)
+		wg.Done()
+	}()
+	// Feed leaves
+	start := time.Now()
+	logged := time.Now()
+	accounts := 0
+	for it.Next() {
+		slimData := it.Account()
+		fullData := SlimToFull(slimData)
+		l := leaf{it.Hash(), fullData}
+		in <- l
+		if time.Since(logged) > 8*time.Second {
+			log.Info("Generating trie hash from snapshot",
+				"at", l.key, "accounts", accounts, "elapsed", time.Since(start))
+			logged = time.Now()
+		}
+		accounts++
+	}
+	close(in)
+	result := <-out
+	log.Info("Generated trie hash from snapshot", "accounts", accounts, "elapsed", time.Since(start))
+	wg.Wait()
+	return result
+}
+
+// ReStackGenerate is a hexary trie builder which is built from the
+// bottom-up as keys are added. It attempts to save memory by doing
+// the RLP encoding on the fly during hashing.
+func ReStackGenerate(in chan (leaf), out chan (common.Hash)) {
+	t := trie.NewReStackTrie()
+	for leaf := range in {
+		t.TryUpdate(leaf.key[:], leaf.value)
+	}
+	out <- t.Hash()
+}
+
+func verifyHasher(it AccountIterator, generatorFn trieGeneratorFn, begin, end int) bool {
+	var (
+		referenceFn = StdGenerate
+
+		inA  = make(chan leaf)        // chan to pass leaves
+		outA = make(chan common.Hash) // chan to collect result
+
+		inB  = make(chan leaf)        // chan to pass leaves
+		outB = make(chan common.Hash) // chan to collect result
+		wg   sync.WaitGroup
+	)
+	wg.Add(2)
+	go func() {
+		referenceFn(inA, outA)
+		wg.Done()
+	}()
+	go func() {
+		generatorFn(inB, outB)
+		wg.Done()
+	}()
+	// Feed leaves
+	start := time.Now()
+	logged := time.Now()
+	accounts := 0
+	for it.Next() {
+		if accounts < begin {
+			accounts++
+			continue
+		}
+		if end > 0 && accounts > end {
+			break
+		}
+		slimData := it.Account()
+		fullData := SlimToFull(slimData)
+		l := leaf{it.Hash(), fullData}
+		inA <- l
+		inB <- l
+		accounts++
+		if time.Since(logged) > 8*time.Second {
+			log.Info("Generating trie hash from snapshot",
+				"at", l.key, "accounts", accounts, "elapsed", time.Since(start))
+			logged = time.Now()
+		}
+	}
+	close(inA)
+	close(inB)
+	resultA := <-outA
+	resultB := <-outB
+	log.Info("Generated trie hash from snapshot", "accounts", accounts,
+		"elapsed", time.Since(start),
+		"start", begin,
+		"end", end,
+		"exp", resultA,
+		"got", resultB)
+	wg.Wait()
+	return resultA == resultB
+}
+
+// PruneGenerate is a hexary trie builder which collapses old nodes, but is still
+// based on more or less the ordinary trie builder
+func PruneGenerate(in chan (leaf), out chan (common.Hash)) {
+	t := trie.NewHashTrie()
+	for leaf := range in {
+		t.TryUpdate(leaf.key[:], leaf.value)
+	}
+	out <- t.Hash()
+}
+
+// StdGenerate is a very basic hexary trie builder which uses the same Trie
+// as the rest of geth, with no enhancements or optimizations
+func StdGenerate(in chan (leaf), out chan (common.Hash)) {
+	t, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New()))
+	for leaf := range in {
+		t.TryUpdate(leaf.key[:], leaf.value)
+	}
+	out <- t.Hash()
+}
+
+// AppendOnlyGenerate is a very basic hexary trie builder which uses the same Trie
+// as the rest of geth, but does not create as many objects while expanding the trie
+func AppendOnlyGenerate(in chan (leaf), out chan (common.Hash)) {
+	t := trie.NewAppendOnlyTrie()
+	for leaf := range in {
+		t.TryUpdate(leaf.key[:], leaf.value)
+	}
+	out <- t.Hash()
+}

core/state/snapshot/iterator_test.go

@@ -84,7 +84,7 @@ func (ti *testIterator) Hash() common.Hash {
 }
 
 func (ti *testIterator) Account() []byte {
-	return nil
+	return []byte{ti.values[0]}
 }
 
 func (ti *testIterator) Release() {}