fix leaf proofs in stateless execution (#22)

* Fixes in witness pre-state

* Add the recipient's nonce to the witness

* reduce PR footprint and investigate issue in root state calculation

* quick build fix

consensus/clique/snapshot_test.go

@@ -403,7 +403,7 @@ func TestClique(t *testing.T) {
 		}
 		// Create a pristine blockchain with the genesis injected
 		db := rawdb.NewMemoryDatabase()
-		genesis.Commit(db, nil)
+		genesis.Commit(db)
 
 		// Assemble a chain of headers from the cast votes
 		config := *params.TestChainConfig

core/chain_makers.go

@@ -320,7 +320,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 					}
 				}
 				vtr.Hash()
-				p, err := vtr.ProveAndSerialize(keys)
+				p, err := vtr.ProveAndSerialize(keys, b.witness.KeyVals())
 				block.SetVerkleProof(p)
 				if err != nil {
 					panic(err)

core/genesis.go

@@ -172,7 +172,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
 		} else {
 			log.Info("Writing custom genesis block")
 		}
-		block, err := genesis.Commit(db, nil)
+		block, err := genesis.Commit(db)
 		if err != nil {
 			return genesis.Config, common.Hash{}, err
 		}
@@ -190,7 +190,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override
 		if hash != stored {
 			return genesis.Config, hash, &GenesisMismatchError{stored, hash}
 		}
-		block, err := genesis.Commit(db, nil)
+		block, err := genesis.Commit(db)
 		if err != nil {
 			return genesis.Config, hash, err
 		}
@@ -317,7 +317,11 @@ func (g *Genesis) ToBlock(db ethdb.Database, snaps *snapshot.Tree) *types.Block
 
 // Commit writes the block and state of a genesis specification to the database.
 // The block is committed as the canonical head block.
-func (g *Genesis) Commit(db ethdb.Database, snaps *snapshot.Tree) (*types.Block, error) {
+func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) {
+	return g.CommitWithSnaps(db, nil)
+}
+
+func (g *Genesis) CommitWithSnaps(db ethdb.Database, snaps *snapshot.Tree) (*types.Block, error) {
 	block := g.ToBlock(db, snaps)
 	if block.Number().Sign() != 0 {
 		return nil, fmt.Errorf("can't commit genesis block with number > 0")
@@ -343,7 +347,7 @@ func (g *Genesis) Commit(db ethdb.Database, snaps *snapshot.Tree) (*types.Block,
 // MustCommit writes the genesis block and state to db, panicking on error.
 // The block is committed as the canonical head block.
 func (g *Genesis) MustCommit(db ethdb.Database) *types.Block {
-	block, err := g.Commit(db, nil)
+	block, err := g.Commit(db)
 	if err != nil {
 		panic(err)
 	}

core/state/statedb.go

@@ -492,6 +492,10 @@ func (s *StateDB) updateStatelessStateObject(obj *stateObject) {
 					copy(b[:], obj.data.Balance.Bytes())
 					binary.BigEndian.PutUint64(cs[:], uint64(len(obj.code)))
 
+					// TODO(@gballet) stateless tree update
+					// i.e. perform a "delta" update on all
+					// commitments. go-verkle currently has
+					// no support for these.
 				}
 			}
 		}

core/state_transition.go

@@ -17,6 +17,7 @@
 package core
 
 import (
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -293,12 +294,23 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) {
 	if st.evm.TxContext.Accesses != nil {
 		if msg.To() != nil {
 			toBalance := trieUtils.GetTreeKeyBalance(*msg.To())
-			gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(toBalance)
+			pre := st.state.GetBalance(*msg.To())
+			gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(toBalance, pre.Bytes())
+
+			// NOTE: Nonce also needs to be charged, because it is needed for execution
+			// on the statless side.
+			var preTN [8]byte
+			fromNonce := trieUtils.GetTreeKeyNonce(*msg.To())
+			binary.BigEndian.PutUint64(preTN[:], st.state.GetNonce(*msg.To()))
+			gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(fromNonce, preTN[:])
 		}
 		fromBalance := trieUtils.GetTreeKeyBalance(msg.From())
+		preFB := st.state.GetBalance(msg.From()).Bytes()
 		fromNonce := trieUtils.GetTreeKeyNonce(msg.From())
-		gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(fromNonce)
-		gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(fromBalance)
+		var preFN [8]byte
+		binary.BigEndian.PutUint64(preFN[:], st.state.GetNonce(msg.From()))
+		gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(fromNonce, preFN[:])
+		gas += st.evm.TxContext.Accesses.TouchAddressAndChargeGas(fromBalance, preFB[:])
 	}
 	st.gas -= gas
 

core/types/access_witness.go

@@ -16,39 +16,47 @@
 
 package types
 
-import "github.com/ethereum/go-ethereum/common"
-import "github.com/ethereum/go-ethereum/params"
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/params"
+)
 
 // AccessWitness lists the locations of the state that are being accessed
 // during the production of a block.
-// TODO(@gballet) this doesn't support deletions
+// TODO(@gballet) this doesn't fully support deletions
 type AccessWitness struct {
-	Witness map[common.Hash]map[byte]struct{}
+	// Branches flags if a given branch has been loaded
+	Branches map[[31]byte]struct{}
+
+	// Chunks contains the initial value of each address
+	Chunks map[common.Hash][]byte
 }
 
 func NewAccessWitness() *AccessWitness {
-	return &AccessWitness{Witness: make(map[common.Hash]map[byte]struct{})}
+	return &AccessWitness{
+		Branches: make(map[[31]byte]struct{}),
+		Chunks:   make(map[common.Hash][]byte),
+	}
 }
 
 // TouchAddress adds any missing addr to the witness and returns respectively
 // true if the stem or the stub weren't arleady present.
-func (aw *AccessWitness) TouchAddress(addr []byte) (bool, bool) {
+func (aw *AccessWitness) TouchAddress(addr, value []byte) (bool, bool) {
 	var (
-		stem        common.Hash
+		stem        [31]byte
 		newStem     bool
 		newSelector bool
-		selector    = addr[31]
 	)
 	copy(stem[:], addr[:31])
 
 	// Check for the presence of the stem
-	if _, newStem := aw.Witness[stem]; !newStem {
-		aw.Witness[stem] = make(map[byte]struct{})
+	if _, newStem := aw.Branches[stem]; !newStem {
+		aw.Branches[stem] = struct{}{}
 	}
 
 	// Check for the presence of the selector
-	if _, newSelector := aw.Witness[stem][selector]; !newSelector {
-		aw.Witness[stem][selector] = struct{}{}
+	if _, newSelector := aw.Chunks[common.BytesToHash(addr)]; !newSelector {
+		aw.Chunks[common.BytesToHash(addr)] = value
 	}
 
 	return newStem, newSelector
@@ -58,10 +66,10 @@ func (aw *AccessWitness) TouchAddress(addr []byte) (bool, bool) {
 // the current witness, and charge extra gas if that isn't the case. This is
 // meant to only be called on a tx-context access witness (i.e. before it is
 // merged), not a block-context witness: witness costs are charged per tx.
-func (aw *AccessWitness) TouchAddressAndChargeGas(addr []byte) uint64 {
+func (aw *AccessWitness) TouchAddressAndChargeGas(addr, value []byte) uint64 {
 	var gas uint64
 
-	nstem, nsel := aw.TouchAddress(addr)
+	nstem, nsel := aw.TouchAddress(addr, value)
 	if nstem {
 		gas += params.WitnessBranchCost
 	}
@@ -75,34 +83,31 @@ func (aw *AccessWitness) TouchAddressAndChargeGas(addr []byte) uint64 {
 // of a tx, with the accumulation of witnesses that were generated during the
 // execution of all the txs preceding this one in a given block.
 func (aw *AccessWitness) Merge(other *AccessWitness) {
-	for k, mo := range other.Witness {
-		// LeafNode-level merge
-		if ma, ok := aw.Witness[k]; ok {
-			for b, y := range mo {
-				// If a particular location isn't already
-				// present, then flag it. The block witness
-				// require only the initial value be present.
-				if _, ok := ma[b]; !ok {
-					ma[b] = y
-				}
-			}
-		} else {
-			aw.Witness[k] = mo
+	for k := range other.Branches {
+		if _, ok := aw.Branches[k]; !ok {
+			aw.Branches[k] = struct{}{}
+		}
+	}
+
+	for k, chunk := range other.Chunks {
+		if _, ok := aw.Chunks[k]; !ok {
+			aw.Chunks[k] = chunk
 		}
 	}
 }
 
 // Key returns, predictably, the list of keys that were touched during the
 // buildup of the access witness.
 func (aw *AccessWitness) Keys() [][]byte {
-	keys := make([][]byte, 0, len(aw.Witness))
-	for stem, branches := range aw.Witness {
-		for selector := range branches {
-			var key [32]byte
-			copy(key[:31], stem[:31])
-			key[31] = selector
-			keys = append(keys, key[:])
-		}
+	keys := make([][]byte, 0, len(aw.Chunks))
+	for key := range aw.Chunks {
+		var k [32]byte
+		copy(k[:], key[:])
+		keys = append(keys, k[:])
 	}
 	return keys
 }
+
+func (aw *AccessWitness) KeyVals() map[common.Hash][]byte {
+	return aw.Chunks
+}

core/vm/gas_table.go

@@ -92,7 +92,9 @@ func gasExtCodeSize(evm *EVM, contract *Contract, stack *Stack, mem *Memory, mem
 	usedGas := uint64(0)
 	slot := stack.Back(0)
 	index := trieUtils.GetTreeKeyCodeSize(common.Address(slot.Bytes20()))
-	usedGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index)
+	// FIXME(@gballet) need to get the exact code size when executing the operation,
+	// the value passed here is invalid.
+	usedGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index, slot.Bytes())
 
 	return usedGas, nil
 }
@@ -122,12 +124,14 @@ func gasCodeCopy(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memory
 		addr := contract.Address()
 		chunk := uint64CodeOffset / 31
 		endChunk := uint64CodeEnd / 31
+		code := evm.StateDB.GetCode(addr)
 		// XXX uint64 overflow in condition check
 		for ; chunk < endChunk; chunk++ {
 
 			// TODO make a version of GetTreeKeyCodeChunk without the bigint
 			index := trieUtils.GetTreeKeyCodeChunk(addr, uint256.NewInt(chunk))
-			statelessGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index)
+			// FIXME(@gballet) invalid code chunk, the jumpdest is missing
+			statelessGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index, code[chunk*31:chunk*31+31])
 		}
 
 	}
@@ -154,12 +158,13 @@ func gasExtCodeCopy(evm *EVM, contract *Contract, stack *Stack, mem *Memory, mem
 		addr := common.Address(a.Bytes20())
 		chunk := uint64CodeOffset / 31
 		endChunk := uint64CodeEnd / 31
+		// TODO(@gballet) implement StateDB::GetCodeChunk()
+		code := evm.StateDB.GetCode(addr)
 		// XXX uint64 overflow in condition check
 		for ; chunk < endChunk; chunk++ {
-
-			// TODO make a version of GetTreeKeyCodeChunk without the bigint
+			// TODO(@gballet) make a version of GetTreeKeyCodeChunk without the bigint
 			index := trieUtils.GetTreeKeyCodeChunk(addr, uint256.NewInt(chunk))
-			statelessGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index)
+			statelessGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index, code[31*chunk:31*(chunk+1)])
 		}
 
 	}
@@ -172,7 +177,8 @@ func gasSLoad(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySiz
 	where := stack.Back(0)
 	addr := contract.Address()
 	index := trieUtils.GetTreeKeyStorageSlot(addr, where)
-	usedGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index)
+	// FIXME(@gballet) invalid value, got to read it from the DB
+	usedGas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index, index)
 
 	return usedGas, nil
 }
@@ -422,7 +428,9 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize
 		// Charge witness costs
 		for i := trieUtils.VersionLeafKey; i <= trieUtils.CodeSizeLeafKey; i++ {
 			index := trieUtils.GetTreeKeyAccountLeaf(address, byte(i))
-			gas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index)
+			// FIXME(@gballet) invalid loaded value - need to introduce a
+			// TouchAccount function.
+			gas += evm.TxContext.Accesses.TouchAddressAndChargeGas(index, index)
 		}
 	}
 

core/vm/interpreter.go

@@ -200,7 +200,8 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (
 		var codePage common.Hash
 		if in.evm.ChainConfig().UseVerkle {
 			index := trieUtils.GetTreeKeyCodeChunk(contract.Address(), uint256.NewInt(pc/31))
-			contract.Gas -= in.evm.TxContext.Accesses.TouchAddressAndChargeGas(index)
+			// FIXME(@gballet) this is only valid when not executing in stateless mode
+			contract.Gas -= in.evm.TxContext.Accesses.TouchAddressAndChargeGas(index, contract.Code[pc&0x1F:pc&0x1F+31])
 
 			if in.evm.accesses != nil {
 				codePage, inWitness = in.evm.accesses[common.BytesToHash(index)]

core/vm/jump_table.go

@@ -426,14 +426,14 @@ func newFrontierInstructionSet() JumpTable {
 		},
 		GASPRICE: {
 			execute:     opGasprice,
-			dynamicGas:  gasExtCodeSize,
 			constantGas: GasQuickStep,
 			minStack:    minStack(0, 1),
 			maxStack:    maxStack(0, 1),
 		},
 		EXTCODESIZE: {
 			execute:     opExtCodeSize,
 			constantGas: params.ExtcodeSizeGasFrontier,
+			dynamicGas:  gasExtCodeSize,
 			minStack:    minStack(1, 1),
 			maxStack:    maxStack(1, 1),
 		},

eth/gasprice/gasprice_test.go

@@ -108,7 +108,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke
 	}
 	engine := ethash.NewFaker()
 	db := rawdb.NewMemoryDatabase()
-	genesis, _ := gspec.Commit(db, nil)
+	genesis, _ := gspec.Commit(db)
 
 	// Generate testing blocks
 	blocks, _ := core.GenerateChain(gspec.Config, genesis, engine, db, testHead+1, func(i int, b *core.BlockGen) {
@@ -145,7 +145,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke
 	})
 	// Construct testing chain
 	diskdb := rawdb.NewMemoryDatabase()
-	gspec.Commit(diskdb, nil)
+	gspec.Commit(diskdb)
 	chain, err := core.NewBlockChain(diskdb, nil, gspec.Config, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to create local chain, %v", err)

tests/block_test_util.go

@@ -106,7 +106,7 @@ func (t *BlockTest) Run(snapshotter bool) error {
 
 	// import pre accounts & construct test genesis block & state root
 	db := rawdb.NewMemoryDatabase()
-	gblock, err := t.genesis(config).Commit(db, nil)
+	gblock, err := t.genesis(config).Commit(db)
 	if err != nil {
 		return err
 	}

trie/verkle.go

@@ -34,6 +34,10 @@ type VerkleTrie struct {
 	db   *Database
 }
 
+func (vt *VerkleTrie) ToDot() string {
+	return verkle.ToDot(vt.root)
+}
+
 func NewVerkleTrie(root verkle.VerkleNode, db *Database) *VerkleTrie {
 	return &VerkleTrie{
 		root: root,
@@ -148,22 +152,19 @@ type verkleproof struct {
 	Leaves []KeyValuePair
 }
 
-func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte) ([]byte, error) {
+func (trie *VerkleTrie) ProveAndSerialize(keys [][]byte, kv map[common.Hash][]byte) ([]byte, error) {
 	d, y, σ := verkle.MakeVerkleMultiProof(trie.root, keys)
 	vp := verkleproof{
 		D: d,
 		Y: y,
 		Σ: σ,
 	}
-	for _, key := range keys {
-		payload, err := trie.TryGet(key)
-		if err != nil {
-			return nil, err
-		}
-
+	for key, val := range kv {
+		var k [32]byte
+		copy(k[:], key[:])
 		vp.Leaves = append(vp.Leaves, KeyValuePair{
-			Key:   key,
-			Value: payload,
+			Key:   k[:],
+			Value: val,
 		})
 	}
 	return rlp.EncodeToBytes(vp)