From 2371185fb57ba0d30ce571ce263967b9f8b65c43 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 8 Sep 2021 18:43:54 +0200 Subject: [PATCH 1/3] test stateless block execution --- core/chain_makers.go | 73 ++++++++++++++++++++++++++++++++- core/state/snapshot/snapshot.go | 12 ++++++ core/state_processor_test.go | 19 ++++----- core/state_transition.go | 11 ++++- core/types/access_witness.go | 38 +++++++++++++++++ core/vm/gas_table.go | 23 +++++------ core/vm/interpreter.go | 5 ++- params/protocol_params.go | 4 ++ trie/verkle.go | 31 +++++++++++--- 9 files changed, 181 insertions(+), 35 deletions(-) diff --git a/core/chain_makers.go b/core/chain_makers.go index ca6f2b442672..95d04caa151c 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" ) // BlockGen creates blocks for testing. @@ -43,6 +44,7 @@ type BlockGen struct { txs []*types.Transaction receipts []*types.Receipt uncles []*types.Header + witness *types.AccessWitness config *params.ChainConfig engine consensus.Engine @@ -103,10 +105,11 @@ func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) { b.SetCoinbase(common.Address{}) } b.statedb.Prepare(tx.Hash(), len(b.txs)) - receipt, _, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{}) + receipt, accesses, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vm.Config{}) if err != nil { panic(err) } + b.witness.Merge(accesses) b.txs = append(b.txs, tx) b.receipts = append(b.receipts, receipt) } @@ -250,6 +253,74 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse return blocks, receipts } +func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) { + if config == nil { + config = params.TestChainConfig + } + blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n) + chainreader := &fakeChainReader{config: config} + genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { + b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine, witness: types.NewAccessWitness()} + b.header = makeHeader(chainreader, parent, statedb, b.engine) + + // Mutate the state and block according to any hard-fork specs + if daoBlock := config.DAOForkBlock; daoBlock != nil { + limit := new(big.Int).Add(daoBlock, params.DAOForkExtraRange) + if b.header.Number.Cmp(daoBlock) >= 0 && b.header.Number.Cmp(limit) < 0 { + if config.DAOForkSupport { + b.header.Extra = common.CopyBytes(params.DAOForkBlockExtra) + } + } + } + if config.DAOForkSupport && config.DAOForkBlock != nil && config.DAOForkBlock.Cmp(b.header.Number) == 0 { + misc.ApplyDAOHardFork(statedb) + } + // Execute any user modifications to the block + if gen != nil { + gen(i, b) + } + if b.engine != nil { + // Generate an associated verkle proof + if tr := statedb.GetTrie(); tr.IsVerkle() { + vtr := tr.(*trie.VerkleTrie) + // Generate the proof if we are using a verkle tree + p, err := vtr.ProveAndSerialize(b.witness.Keys()) + b.header.VerkleProof = p + if err != nil { + panic(err) + } + } + // Finalize and seal the block + block, err := b.engine.FinalizeAndAssemble(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts) + if err != nil { + panic(err) + } + + // Write state changes to db + root, err := statedb.Commit(config.IsEIP158(b.header.Number)) + if err != nil { + panic(fmt.Sprintf("state write error: %v", err)) + } + if err := statedb.Database().TrieDB().Commit(root, false, nil); err != nil { + panic(fmt.Sprintf("trie write error: %v", err)) + } + return block, b.receipts + } + return nil, nil + } + for i := 0; i < n; i++ { + statedb, err := state.New(parent.Root(), state.NewDatabaseWithConfig(db, &trie.Config{UseVerkle: true}), nil) + if err != nil { + panic(err) + } + block, receipt := genblock(i, parent, statedb) + blocks[i] = block + receipts[i] = receipt + parent = block + } + return blocks, receipts +} + func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { var time uint64 if parent.Time() == 0 { diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 46c41d1d5964..3e2563ee717f 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -24,6 +24,7 @@ import ( "sync" "sync/atomic" + "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" @@ -193,6 +194,17 @@ func New(diskdb ethdb.KeyValueStore, triedb *trie.Database, cache int, root comm } if err != nil { if rebuild { + if useVerkle { + snap.layers = map[common.Hash]snapshot{ + root: &diskLayer{ + diskdb: diskdb, + triedb: triedb, + root: root, + cache: fastcache.New(cache * 1024 * 1024), + }, + } + return snap, nil + } log.Warn("Failed to load snapshot, regenerating", "err", err) snap.Rebuild(root) return snap, nil diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 23db822031b4..8bec46be4bc9 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -324,18 +324,13 @@ func TestProcessStateless(t *testing.T) { genesis := gspec.MustCommit(db, nil) blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() - var makeTx = func(nonce uint64, to common.Address, amount *big.Int, gasLimit uint64, gasPrice *big.Int, data []byte) *types.Transaction { - tx, _ := types.SignTx(types.NewTransaction(nonce, to, amount, gasLimit, gasPrice, data), signer, testKey) - return tx - } - bigNumber := new(big.Int).SetBytes(common.FromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) - tooBigNumber := new(big.Int).Set(bigNumber) - tooBigNumber.Add(tooBigNumber, common.Big1) - txs := []*types.Transaction{ - makeTx(0, common.Address{}, big.NewInt(0), params.TxGas, big.NewInt(875000000), nil), - } - block := GenerateBadBlock(genesis, ethash.NewFaker(), txs, gspec.Config) - _, err := blockchain.InsertChain(types.Blocks{block}) + chain, _ := GenerateVerkleChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(_ int, gen *BlockGen) { + toaddr := common.Address{} + tx, _ := types.SignTx(types.NewTransaction(0, toaddr, big.NewInt(0), params.TxGas, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + + }) + _, err := blockchain.InsertChain(chain) if err != nil { t.Fatalf("block imported with error: %v", err) } diff --git a/core/state_transition.go b/core/state_transition.go index b5b0ae990a41..0ddf8417e56c 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" + trieUtils "github.com/ethereum/go-ethereum/trie/utils" ) /* @@ -112,7 +113,7 @@ func (result *ExecutionResult) Revert() []byte { } // IntrinsicGas computes the 'intrinsic gas' for a message with the given data. -func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool) (uint64, error) { +func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation, isHomestead, isEIP2028 bool) (uint64, error) { // Set the starting gas for the raw transaction var gas uint64 if isContractCreation && isHomestead { @@ -289,6 +290,14 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { if st.gas < gas { return nil, fmt.Errorf("%w: have %d, want %d", ErrIntrinsicGas, st.gas, gas) } + if st.evm.TxContext.Accesses != nil { + toBalance := trieUtils.GetTreeKeyBalance(*msg.To()) + fromBalance := trieUtils.GetTreeKeyBalance(msg.From()) + fromNonce := trieUtils.GetTreeKeyNonce(msg.From()) + gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(toBalance) + gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(fromNonce) + gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(fromBalance) + } st.gas -= gas // Check clause 6 diff --git a/core/types/access_witness.go b/core/types/access_witness.go index ce504779d6ed..1d4c717a1151 100644 --- a/core/types/access_witness.go +++ b/core/types/access_witness.go @@ -17,6 +17,7 @@ package types import "github.com/ethereum/go-ethereum/common" +import "github.com/ethereum/go-ethereum/params" type AccessWitness struct { Witness map[common.Hash]map[byte]struct{} @@ -26,6 +27,43 @@ func NewAccessWitness() *AccessWitness { return &AccessWitness{Witness: make(map[common.Hash]map[byte]struct{})} } +// TouchAddress adds any missing addr to the witness and returns respectively +// true if the stem or the stub weren't arleady present. +func (aw *AccessWitness) TouchAddress(addr []byte) (bool, bool) { + var ( + stem common.Hash + newStem bool + newSelector bool + selector = addr[31] + ) + copy(stem[:], addr[:31]) + + // Check for the presence of the stem + if _, newStem := aw.Witness[stem]; !newStem { + aw.Witness[stem] = make(map[byte]struct{}) + } + + // Check for the presence of the selector + if _, newSelector := aw.Witness[stem][selector]; !newSelector { + aw.Witness[stem][selector] = struct{}{} + } + + return newStem, newSelector +} + +func (aw *AccessWitness) TouchAddressAndChargeGas(addr []byte) uint64 { + var gas uint64 + + nstem, nsel := aw.TouchAddress(addr) + if nstem { + gas += params.WitnessBranchCost + } + if nsel { + gas += params.WitnessChunkCost + } + return gas +} + func (aw *AccessWitness) Merge(other *AccessWitness) { for k, mo := range other.Witness { if ma, ok := aw.Witness[k]; ok { diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index e23b40abb9da..e5689248ec66 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -97,12 +97,12 @@ func gasExtCodeSize(evm *EVM, contract *Contract, stack *Stack, mem *Memory, mem _, ok := evm.TxContext.Accesses.Witness[subtree] if !ok { evm.TxContext.Accesses.Witness[subtree] = make(map[byte]struct{}) - usedGas += gasWitnessBranchCost + usedGas += params.WitnessBranchCost } _, ok = evm.TxContext.Accesses.Witness[subtree][subleaf] if !ok { - usedGas += gasWitnessChunkCost + usedGas += params.WitnessChunkCost evm.TxContext.Accesses.Witness[subtree][subleaf] = struct{}{} } @@ -114,9 +114,6 @@ var ( gasCodeCopyStateful = memoryCopierGas(2) gasExtCodeCopyStateful = memoryCopierGas(3) gasReturnDataCopy = memoryCopierGas(2) - - gasWitnessBranchCost = uint64(1900) - gasWitnessChunkCost = uint64(200) ) func gasCodeCopy(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { @@ -147,12 +144,12 @@ func gasCodeCopy(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memory _, ok := evm.TxContext.Accesses.Witness[subtree] if !ok { evm.TxContext.Accesses.Witness[subtree] = make(map[byte]struct{}) - statelessGas += gasWitnessBranchCost + statelessGas += params.WitnessBranchCost } _, ok = evm.TxContext.Accesses.Witness[subtree][subleaf] if !ok { - statelessGas += gasWitnessChunkCost + statelessGas += params.WitnessChunkCost evm.TxContext.Accesses.Witness[subtree][subleaf] = struct{}{} } @@ -192,12 +189,12 @@ func gasExtCodeCopy(evm *EVM, contract *Contract, stack *Stack, mem *Memory, mem _, ok := evm.TxContext.Accesses.Witness[subtree] if !ok { evm.TxContext.Accesses.Witness[subtree] = make(map[byte]struct{}) - statelessGas += gasWitnessBranchCost + statelessGas += params.WitnessBranchCost } _, ok = evm.TxContext.Accesses.Witness[subtree][subleaf] if !ok { - statelessGas += gasWitnessChunkCost + statelessGas += params.WitnessChunkCost evm.TxContext.Accesses.Witness[subtree][subleaf] = struct{}{} } @@ -218,12 +215,12 @@ func gasSLoad(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySiz _, ok := evm.TxContext.Accesses.Witness[subtree] if !ok { evm.TxContext.Accesses.Witness[subtree] = make(map[byte]struct{}) - usedGas += gasWitnessBranchCost + usedGas += params.WitnessBranchCost } _, ok = evm.TxContext.Accesses.Witness[subtree][subleaf] if !ok { - usedGas += gasWitnessChunkCost + usedGas += params.WitnessChunkCost evm.TxContext.Accesses.Witness[subtree][subleaf] = struct{}{} } @@ -480,12 +477,12 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize _, ok := evm.TxContext.Accesses.Witness[subtree] if !ok { evm.TxContext.Accesses.Witness[subtree] = make(map[byte]struct{}) - gas += gasWitnessBranchCost + gas += params.WitnessBranchCost } _, ok = evm.TxContext.Accesses.Witness[subtree][subleaf] if !ok { - gas += gasWitnessChunkCost + gas += params.WitnessChunkCost evm.TxContext.Accesses.Witness[subtree][subleaf] = struct{}{} } } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index d8b6f72f868e..6d90b4b561f0 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" trieUtils "github.com/ethereum/go-ethereum/trie/utils" "github.com/holiman/uint256" ) @@ -204,11 +205,11 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( subleaf := index[31] if _, ok := in.evm.TxContext.Accesses.Witness[subtree]; !ok { in.evm.TxContext.Accesses.Witness[subtree] = make(map[byte]struct{}) - contract.Gas -= gasWitnessBranchCost + contract.Gas -= params.WitnessBranchCost } if _, ok := in.evm.TxContext.Accesses.Witness[subtree][subleaf]; !ok { in.evm.TxContext.Accesses.Witness[subtree][subleaf] = struct{}{} - contract.Gas -= gasWitnessChunkCost + contract.Gas -= params.WitnessChunkCost } if in.evm.accesses != nil { diff --git a/params/protocol_params.go b/params/protocol_params.go index 7abb2441bf30..fc0e83a18a01 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -156,6 +156,10 @@ const ( // up to half the consumed gas could be refunded. Redefined as 1/5th in EIP-3529 RefundQuotient uint64 = 2 RefundQuotientEIP3529 uint64 = 5 + + // Verkle tree EIP: costs associated to witness accesses + WitnessBranchCost = uint64(1900) + WitnessChunkCost = uint64(200) ) // Gas discount table for BLS12-381 G1 and G2 multi exponentiation operations diff --git a/trie/verkle.go b/trie/verkle.go index c41028b90a09..1577f59d0263 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -57,7 +57,9 @@ func (trie *VerkleTrie) TryGet(key []byte) ([]byte, error) { // by the caller while they are stored in the trie. If a node was not found in the // database, a trie.MissingNodeError is returned. func (trie *VerkleTrie) TryUpdate(key, value []byte) error { - return trie.root.Insert(key, value) + return trie.root.Insert(key, value, func(h []byte) ([]byte, error) { + return trie.db.DiskDB().Get(h) + }) } // TryDelete removes any existing value for key from the trie. If a node was not @@ -131,31 +133,48 @@ func (trie *VerkleTrie) IsVerkle() bool { return true } +type KeyValuePair struct { + Key []byte + Value []byte +} + type verkleproof struct { D *bls.G1Point Y *bls.Fr Σ *bls.G1Point - Leaves map[common.Hash]common.Hash + Leaves []KeyValuePair } func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte) ([]byte, error) { d, y, σ := verkle.MakeVerkleMultiProof(trie.root, keys) - leaves := make(map[common.Hash]common.Hash) + vp := verkleproof{ + D: d, + Y: y, + Σ: σ, + } for _, key := range keys { payload, err := trie.TryGet(key) if err != nil { return nil, err } - leaves[common.Hash{}] = common.BytesToHash(payload) + + vp.Leaves = append(vp.Leaves, KeyValuePair{ + Key: key, + Value: payload, + }) } - return rlp.EncodeToBytes(verkleproof{D: d, Y: y, Σ: σ, Leaves: leaves}) + return rlp.EncodeToBytes(vp) } func DeserializeVerkleProof(proof []byte) (*bls.G1Point, *bls.Fr, *bls.G1Point, map[common.Hash]common.Hash) { var vp verkleproof + var leaves map[common.Hash]common.Hash rlp.DecodeBytes(proof, &vp) - return vp.D, vp.Y, vp.Σ, vp.Leaves + for _, kvp := range vp.Leaves { + leaves[common.BytesToHash(kvp.Key)] = common.BytesToHash(kvp.Value) + } + return vp.D, vp.Y, vp.Σ, leaves } func ChunkifyCode(addr common.Address, code []byte) ([][32]byte, error) { From df2e0b3517ee9d62bb65dd94350ec1d7b5f72134 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 10 Sep 2021 13:40:04 +0200 Subject: [PATCH 2/3] Force tree resolution before generating the proof --- core/chain_makers.go | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/core/chain_makers.go b/core/chain_makers.go index 95d04caa151c..7ce9d9a0d86e 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -280,16 +280,6 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine gen(i, b) } if b.engine != nil { - // Generate an associated verkle proof - if tr := statedb.GetTrie(); tr.IsVerkle() { - vtr := tr.(*trie.VerkleTrie) - // Generate the proof if we are using a verkle tree - p, err := vtr.ProveAndSerialize(b.witness.Keys()) - b.header.VerkleProof = p - if err != nil { - panic(err) - } - } // Finalize and seal the block block, err := b.engine.FinalizeAndAssemble(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts) if err != nil { @@ -304,6 +294,32 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine if err := statedb.Database().TrieDB().Commit(root, false, nil); err != nil { panic(fmt.Sprintf("trie write error: %v", err)) } + + // Generate an associated verkle proof + if tr := statedb.GetTrie(); tr.IsVerkle() { + vtr := tr.(*trie.VerkleTrie) + // Generate the proof if we are using a verkle tree + // WORKAROUND: make sure all keys are resolved + // before building the proof. Ultimately, node + // resolution can be done with a prefetcher or + // from GetCommitmentsAlongPath. + keys := b.witness.Keys() + for _, key := range keys { + out, err := vtr.TryGet(key) + if err != nil { + panic(err) + } + if len(out) == 0 { + panic(fmt.Sprintf("%x should be present in the tree", key)) + } + } + vtr.Hash() + p, err := vtr.ProveAndSerialize(keys) + b.header.VerkleProof = p + if err != nil { + panic(err) + } + } return block, b.receipts } return nil, nil From 1fbc0fec654f50ad84afddce3ca85b1e96df9597 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 10 Sep 2021 17:51:43 +0200 Subject: [PATCH 3/3] increase coverage in stateless test execution --- core/state/statedb.go | 2 +- core/state_processor_test.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index cdcec5260616..47f30f9d632e 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -514,7 +514,7 @@ func (s *StateDB) deleteStateObject(obj *stateObject) { defer func(start time.Time) { s.AccountUpdates += time.Since(start) }(time.Now()) } // Delete the account from the trie - if false { + if !s.trie.IsVerkle() { addr := obj.Address() if err := s.trie.TryDelete(addr[:]); err != nil { s.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err)) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 8bec46be4bc9..f4da1f44f8e2 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -325,8 +325,11 @@ func TestProcessStateless(t *testing.T) { blockchain, _ := NewBlockChain(db, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) defer blockchain.Stop() chain, _ := GenerateVerkleChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(_ int, gen *BlockGen) { - toaddr := common.Address{} - tx, _ := types.SignTx(types.NewTransaction(0, toaddr, big.NewInt(0), params.TxGas, big.NewInt(875000000), nil), signer, testKey) + tx, _ := types.SignTx(types.NewTransaction(0, common.Address{1, 2, 3}, big.NewInt(999), params.TxGas, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + tx, _ = types.SignTx(types.NewTransaction(1, common.Address{}, big.NewInt(999), params.TxGas, big.NewInt(875000000), nil), signer, testKey) + gen.AddTx(tx) + tx, _ = types.SignTx(types.NewTransaction(2, common.Address{}, big.NewInt(0), params.TxGas, big.NewInt(875000000), nil), signer, testKey) gen.AddTx(tx) })