From e6a6368cfda929f41c96e1ca2548dac038000a27 Mon Sep 17 00:00:00 2001 From: Shogo Hyodo Date: Tue, 13 Jun 2023 02:13:34 +0900 Subject: [PATCH] Revert "feat: Deep tree structure with Updates (#13)" This reverts commit 4b7852cc3b70355c495853aee93a141e290c19fb. --- .golangci.yml | 2 +- deepsubtree.go | 421 -------------------------------------------- deepsubtree_test.go | 115 ------------ nodedb.go | 1 + 4 files changed, 2 insertions(+), 537 deletions(-) delete mode 100644 deepsubtree.go delete mode 100644 deepsubtree_test.go diff --git a/.golangci.yml b/.golangci.yml index 66682cc5c..3ed11ffc0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -71,4 +71,4 @@ linters-settings: allow-unused: false allow-leading-space: true require-explanation: false - require-specific: false \ No newline at end of file + require-specific: false diff --git a/deepsubtree.go b/deepsubtree.go deleted file mode 100644 index 4563cfb6f..000000000 --- a/deepsubtree.go +++ /dev/null @@ -1,421 +0,0 @@ -package iavl - -import ( - "bytes" - "encoding/binary" - "errors" - "fmt" - "strings" - - ics23 "github.com/confio/ics23/go" -) - -const ( - // lengthByte is the length prefix prepended to each of the sha256 sub-hashes - lengthByte byte = 0x20 -) - -// Represents a IAVL Deep Subtree that can contain -// a subset of nodes of an IAVL tree -type DeepSubTree struct { - *MutableTree -} - -// Traverses the nodes in the NodeDB that are part of Deep Subtree -// and links them together using the populated left and right -// hashes and sets the root to be the node with the given rootHash -func (dst *DeepSubTree) BuildTree(rootHash []byte) error { - if dst.root == nil { - rootNode, rootErr := dst.ndb.GetNode(rootHash) - if rootErr != nil { - return fmt.Errorf("could not set root of deep subtree: %w", rootErr) - } - dst.root = rootNode - } else if !bytes.Equal(dst.root.hash, rootHash) { - return fmt.Errorf( - "deep Subtree rootHash: %s does not match expected rootHash: %s", - dst.root.hash, - rootHash, - ) - } - nodes, traverseErr := dst.ndb.nodes() - if traverseErr != nil { - return fmt.Errorf("could not traverse nodedb: %w", traverseErr) - } - for _, node := range nodes { - pnode, err := dst.ndb.GetNode(node.hash) - if err != nil { - return err - } - err = dst.linkNode(pnode) - if err != nil { - return err - } - } - // Now that nodes are linked correctly, traverse again - // and set their keys correctly - for _, node := range nodes { - pnode, err := dst.ndb.GetNode(node.hash) - if err != nil { - return err - } - if pnode.leftNode != nil { - pnode.key = pnode.leftNode.getHighestKey() - } - - if pnode.rightNode != nil { - pnode.key = pnode.rightNode.getLowestKey() - } - } - - return nil -} - -// Link the given node if it is not linked yet -// If already linked, return an error in case connection was made incorrectly -// Note: GetNode returns nil if the node with the hash passed into it does not exist -// which is expected with a deep subtree. -func (dst *DeepSubTree) linkNode(node *Node) error { - if len(node.leftHash) > 0 { - if node.leftNode == nil { - node.leftNode, _ = dst.ndb.GetNode(node.leftHash) - } else if !bytes.Equal(node.leftNode.hash, node.leftHash) { - return fmt.Errorf( - "for node: %s, leftNode hash: %s and node leftHash: %s do not match", - node.hash, - node.leftNode.hash, - node.leftHash, - ) - } - } - if len(node.rightHash) > 0 { - if node.rightNode == nil { - node.rightNode, _ = dst.ndb.GetNode(node.rightHash) - } else if !bytes.Equal(node.rightNode.hash, node.rightHash) { - return fmt.Errorf( - "for node: %s, rightNode hash: %s and node rightHash: %s do not match", - node.hash, - node.rightNode.hash, - node.rightHash, - ) - } - } - return nil -} - -// Set sets a key in the working tree with the given value. -// Assumption: Node with given key already exists and is a leaf node. -// Modified version of set taken from mutable_tree.go -func (dst *DeepSubTree) Set(key []byte, value []byte) (updated bool, err error) { - if value == nil { - return updated, fmt.Errorf("attempt to store nil value at key '%s'", key) - } - - dst.root, updated, err = dst.recursiveSet(dst.root, key, value) - if err != nil { - return updated, err - } - return updated, recomputeHash(dst.root) -} - -func recomputeHash(node *Node) error { - node.hash = nil - _, err := node._hash() - return err -} - -// Helper method for set to traverse and find the node with given key -// recursively. -func (dst *DeepSubTree) recursiveSet(node *Node, key []byte, value []byte) ( - newSelf *Node, updated bool, err error, -) { - version := dst.version + 1 - - if node.isLeaf() { - if !bytes.Equal(key, node.key) { - return nil, false, fmt.Errorf("adding new keys is not supported") - } - return NewNode(key, value, version), true, nil - } - // Otherwise, node is inner node - node.version = version - leftNode, rightNode := node.leftNode, node.rightNode - if leftNode == nil && rightNode == nil { - return nil, false, fmt.Errorf("inner node must have at least one child node set") - } - compare := bytes.Compare(key, node.key) - switch { - case leftNode != nil && (compare < 0 || rightNode == nil): - node.leftNode, updated, err = dst.recursiveSet(leftNode, key, value) - if err != nil { - return nil, updated, err - } - hashErr := recomputeHash(node.leftNode) - if hashErr != nil { - return nil, updated, hashErr - } - node.leftHash = node.leftNode.hash - case rightNode != nil && (compare >= 0 || leftNode == nil): - node.rightNode, updated, err = dst.recursiveSet(rightNode, key, value) - if err != nil { - return nil, updated, err - } - hashErr := recomputeHash(node.rightNode) - if hashErr != nil { - return nil, updated, hashErr - } - node.rightHash = node.rightNode.hash - default: - return nil, false, fmt.Errorf("inner node does not have key set correctly") - } - return node, updated, nil -} - -// nolint: unused -// Prints a Deep Subtree recursively. -// Modified version of printNode from util.go -func (dst *DeepSubTree) printNodeDeepSubtree(node *Node, indent int) error { - indentPrefix := strings.Repeat(" ", indent) - - if node == nil { - fmt.Printf("%s\n", indentPrefix) - return nil - } - if node.rightNode != nil { - err := dst.printNodeDeepSubtree(node.rightNode, indent+1) - if err != nil { - return err - } - } - - hash, err := node._hash() - if err != nil { - return err - } - - fmt.Printf("%sh:%X\n", indentPrefix, hash) - if node.isLeaf() { - fmt.Printf("%s%X:%X (%v)\n", indentPrefix, node.key, node.value, node.height) - } - - if node.leftNode != nil { - err := dst.printNodeDeepSubtree(node.leftNode, indent+1) - if err != nil { - return err - } - } - return nil -} - -// Returns the highest key in the node's subtree -func (node *Node) getHighestKey() []byte { - if node.isLeaf() { - return node.key - } - highestKey := []byte{} - if node.rightNode != nil { - highestKey = node.rightNode.getHighestKey() - } - if node.leftNode != nil { - leftHighestKey := node.leftNode.getHighestKey() - if len(highestKey) == 0 { - highestKey = leftHighestKey - } else if string(leftHighestKey) > string(highestKey) { - highestKey = leftHighestKey - } - } - return highestKey -} - -// Returns the lowest key in the node's subtree -func (node *Node) getLowestKey() []byte { - if node.isLeaf() { - return node.key - } - lowestKey := []byte{} - if node.rightNode != nil { - lowestKey = node.rightNode.getLowestKey() - } - if node.leftNode != nil { - leftLowestKey := node.leftNode.getLowestKey() - if len(lowestKey) == 0 { - lowestKey = leftLowestKey - } else if string(leftLowestKey) < string(lowestKey) { - lowestKey = leftLowestKey - } - } - return lowestKey -} - -func (dst *DeepSubTree) AddProof(proof *ics23.CommitmentProof) error { - proof = ics23.Decompress(proof) - - if exist := proof.GetExist(); exist != nil { - err := dst.addExistenceProof(exist) - if err != nil { - return err - } - } else if nonExist := proof.GetNonexist(); nonExist != nil { - err := dst.addExistenceProof(nonExist.Left) - if err != nil { - return err - } - err = dst.addExistenceProof(nonExist.Right) - if err != nil { - return err - } - } - - return dst.ndb.Commit() -} - -func (dst *DeepSubTree) addExistenceProof(proof *ics23.ExistenceProof) error { - leaf, err := fromLeafOp(proof.GetLeaf(), proof.Key, proof.Value) - if err != nil { - return err - } - err = dst.ndb.SaveNode(leaf) - if err != nil { - return err - } - prevHash := leaf.hash - path := proof.GetPath() - for i := range path { - inner, err := fromInnerOp(path[i], prevHash) - if err != nil { - return err - } - prevHash = inner.hash - - has, err := dst.ndb.Has(inner.hash) - if err != nil { - return err - } - if !has { - err = dst.ndb.SaveNode(inner) - if err != nil { - return err - } - } - } - return nil -} - -func fromLeafOp(lop *ics23.LeafOp, key, value []byte) (*Node, error) { - r := bytes.NewReader(lop.Prefix) - height, err := binary.ReadVarint(r) - if err != nil { - return nil, err - } - if height != 0 { - return nil, errors.New("height should be 0 in the leaf") - } - size, err := binary.ReadVarint(r) - if err != nil { - return nil, err - } - if size != 1 { - return nil, errors.New("size should be 1 in the leaf") - } - version, err := binary.ReadVarint(r) - if err != nil { - return nil, err - } - node := &Node{ - key: key, - value: value, - size: size, - version: version, - } - - _, err = node._hash() - if err != nil { - return nil, err - } - - return node, nil -} - -func fromInnerOp(iop *ics23.InnerOp, prevHash []byte) (*Node, error) { - r := bytes.NewReader(iop.Prefix) - height, err := binary.ReadVarint(r) - if err != nil { - return nil, err - } - size, err := binary.ReadVarint(r) - if err != nil { - return nil, err - } - version, err := binary.ReadVarint(r) - if err != nil { - return nil, err - } - - b, err := r.ReadByte() - if err != nil { - return nil, err - } - if b != lengthByte { - return nil, errors.New("expected length byte (0x20") - } - var left, right []byte - // if left is empty, skip to right - if r.Len() != 0 { - left = make([]byte, lengthByte) - n, err := r.Read(left) - if err != nil { - return nil, err - } - if n != 32 { - return nil, errors.New("couldn't read left hash") - } - b, err = r.ReadByte() - if err != nil { - return nil, err - } - if b != lengthByte { - return nil, errors.New("expected length byte (0x20") - } - } - - if len(iop.Suffix) > 0 { - right = make([]byte, lengthByte) - r = bytes.NewReader(iop.Suffix) - b, err := r.ReadByte() - if err != nil { - return nil, err - } - if b != lengthByte { - return nil, errors.New("expected length byte (0x20") - } - - n, err := r.Read(right) - if err != nil { - return nil, err - } - if n != 32 { - return nil, errors.New("couldn't read right hash") - } - } - - if left == nil { - left = prevHash - } else if right == nil { - right = prevHash - } - - node := &Node{ - leftHash: left, - rightHash: right, - version: version, - size: size, - height: int8(height), - } - - _, err = node._hash() - if err != nil { - return nil, err - } - - return node, nil -} diff --git a/deepsubtree_test.go b/deepsubtree_test.go deleted file mode 100644 index 3a4d330fc..000000000 --- a/deepsubtree_test.go +++ /dev/null @@ -1,115 +0,0 @@ -package iavl - -import ( - "testing" - - "github.com/stretchr/testify/require" - db "github.com/tendermint/tm-db" -) - -// Tests creating a Deep Subtree step by step -// as a full IAVL tree and checks if roots are equal -func TestDeepSubtreeStepByStep(t *testing.T) { - require := require.New(t) - getTree := func() *MutableTree { - tree, err := getTestTree(5) - require.NoError(err) - - tree.Set([]byte("e"), []byte{5}) - tree.Set([]byte("d"), []byte{4}) - tree.Set([]byte("c"), []byte{3}) - tree.Set([]byte("b"), []byte{2}) - tree.Set([]byte("a"), []byte{1}) - - _, _, err = tree.SaveVersion() - require.NoError(err) - return tree - } - - tree := getTree() - rootHash := tree.root.hash - - mutableTree, err := NewMutableTree(db.NewMemDB(), 100, false) - require.NoError(err) - dst := DeepSubTree{mutableTree} - - // insert key/value pairs in tree - allkeys := [][]byte{ - []byte("a"), []byte("b"), []byte("c"), []byte("d"), []byte("e"), - } - - // Put all keys inside the tree one by one - for _, key := range allkeys { - ics23proof, err := tree.GetMembershipProof(key) - require.NoError(err) - err = dst.AddProof(ics23proof) - require.NoError(err) - - err = dst.BuildTree(rootHash) - require.NoError(err) - } - - // Check root hashes are equal - require.Equal(dst.root.hash, tree.root.hash) -} - -// Tests updating the deepsubtree returns the -// correct roots -// Reference: https://ethresear.ch/t/data-availability-proof-friendly-state-tree-transitions/1453/23 -func TestDeepSubtreeWithUpdates(t *testing.T) { - require := require.New(t) - getTree := func() *MutableTree { - tree, err := getTestTree(5) - require.NoError(err) - - tree.Set([]byte("e"), []byte{5}) - tree.Set([]byte("d"), []byte{4}) - tree.Set([]byte("c"), []byte{3}) - tree.Set([]byte("b"), []byte{2}) - tree.Set([]byte("a"), []byte{1}) - - _, _, err = tree.SaveVersion() - require.NoError(err) - return tree - } - - testCases := [][][]byte{ - { - []byte("a"), []byte("b"), - }, - { - []byte("c"), []byte("d"), - }, - } - - for _, subsetKeys := range testCases { - tree := getTree() - rootHash := tree.root.hash - mutableTree, err := NewMutableTree(db.NewMemDB(), 100, false) - require.NoError(err) - dst := DeepSubTree{mutableTree} - for _, subsetKey := range subsetKeys { - ics23proof, err := tree.GetMembershipProof(subsetKey) - require.NoError(err) - err = dst.AddProof(ics23proof) - require.NoError(err) - dst.BuildTree(rootHash) - require.NoError(err) - } - dst.SaveVersion() - - // Check root hashes are equal - require.Equal(dst.root.hash, tree.root.hash) - - values := [][]byte{{10}, {20}} - for i, subsetKey := range subsetKeys { - dst.Set(subsetKey, values[i]) - dst.SaveVersion() - tree.Set(subsetKey, values[i]) - tree.SaveVersion() - } - - // Check root hashes are equal - require.Equal(dst.root.hash, tree.root.hash) - } -} diff --git a/nodedb.go b/nodedb.go index fd513d75a..602dbcec2 100644 --- a/nodedb.go +++ b/nodedb.go @@ -981,6 +981,7 @@ func (ndb *nodeDB) leafNodes() ([]*Node, error) { return leaves, nil } +// nolint: unused func (ndb *nodeDB) nodes() ([]*Node, error) { nodes := []*Node{}