From 8ffd5bf65224a706ffcdc4ed8b2ee272f244233a Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sat, 9 Dec 2017 06:43:50 -0800 Subject: [PATCH] Clone w/ version; Remove orphanIndex --- .gitignore | 2 + basic_test.go | 22 ++++----- logger.go | 11 +++++ node.go | 58 ++++++++++++----------- nodedb.go | 117 +++++++++++++++++----------------------------- orphaning_tree.go | 32 ++++++------- proof.go | 44 +++++++++-------- proof_range.go | 24 +++++----- tree.go | 33 +++++++++++-- tree_test.go | 48 +++---------------- versioned_tree.go | 54 ++++----------------- 11 files changed, 191 insertions(+), 254 deletions(-) create mode 100644 logger.go diff --git a/.gitignore b/.gitignore index 65dd13daf..dff7e1bb1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ vendor .glide +*.swp +*.swo # created in test code test.db diff --git a/basic_test.go b/basic_test.go index f170708c6..0f091e78e 100644 --- a/basic_test.go +++ b/basic_test.go @@ -32,7 +32,7 @@ func TestBasic(t *testing.T) { // Test 0x00 { - idx, val := tree.Get([]byte{0x00}) + idx, val := tree.Get64([]byte{0x00}) if val != nil { t.Errorf("Expected no value to exist") } @@ -46,7 +46,7 @@ func TestBasic(t *testing.T) { // Test "1" { - idx, val := tree.Get([]byte("1")) + idx, val := tree.Get64([]byte("1")) if val == nil { t.Errorf("Expected value to exist") } @@ -60,7 +60,7 @@ func TestBasic(t *testing.T) { // Test "2" { - idx, val := tree.Get([]byte("2")) + idx, val := tree.Get64([]byte("2")) if val == nil { t.Errorf("Expected value to exist") } @@ -74,7 +74,7 @@ func TestBasic(t *testing.T) { // Test "4" { - idx, val := tree.Get([]byte("4")) + idx, val := tree.Get64([]byte("4")) if val != nil { t.Errorf("Expected no value to exist") } @@ -88,7 +88,7 @@ func TestBasic(t *testing.T) { // Test "6" { - idx, val := tree.Get([]byte("6")) + idx, val := tree.Get64([]byte("6")) if val != nil { t.Errorf("Expected no value to exist") } @@ -103,7 +103,7 @@ func TestBasic(t *testing.T) { func TestUnit(t *testing.T) { - expectHash := func(tree *Tree, hashCount int) { + expectHash := func(tree *Tree, hashCount int64) { // ensure number of new hash calculations is as expected. hash, count := tree.hashWithCount() if count != hashCount { @@ -121,7 +121,7 @@ func TestUnit(t *testing.T) { } } - expectSet := func(tree *Tree, i int, repr string, hashCount int) { + expectSet := func(tree *Tree, i int, repr string, hashCount int64) { origNode := tree.root updated := tree.Set(i2b(i), []byte{}) // ensure node was added & structure is as expected. @@ -134,7 +134,7 @@ func TestUnit(t *testing.T) { tree.root = origNode } - expectRemove := func(tree *Tree, i int, repr string, hashCount int) { + expectRemove := func(tree *Tree, i int, repr string, hashCount int64) { origNode := tree.root value, removed := tree.Remove(i2b(i)) // ensure node was added & structure is as expected. @@ -249,7 +249,7 @@ func TestIntegration(t *testing.T) { if has := tree.Has([]byte(randstr(12))); has { t.Error("Table has extra key") } - if _, val := tree.Get([]byte(r.key)); string(val) != string(r.value) { + if _, val := tree.Get64([]byte(r.key)); string(val) != string(r.value) { t.Error("wrong value") } } @@ -267,7 +267,7 @@ func TestIntegration(t *testing.T) { if has := tree.Has([]byte(randstr(12))); has { t.Error("Table has extra key") } - _, val := tree.Get([]byte(r.key)) + _, val := tree.Get64([]byte(r.key)) if string(val) != string(r.value) { t.Error("wrong value") } @@ -379,7 +379,7 @@ func TestPersistence(t *testing.T) { t2 := NewVersionedTree(0, db) t2.Load() for key, value := range records { - _, t2value := t2.Get([]byte(key)) + _, t2value := t2.Get64([]byte(key)) if string(t2value) != value { t.Fatalf("Invalid value. Expected %v, got %v", value, t2value) } diff --git a/logger.go b/logger.go new file mode 100644 index 000000000..b53ce34ed --- /dev/null +++ b/logger.go @@ -0,0 +1,11 @@ +package iavl + +import ( + "fmt" +) + +func debug(format string, args ...interface{}) { + if false { + fmt.Printf(format, args...) + } +} diff --git a/node.go b/node.go index 7ea246222..91e51f2a7 100644 --- a/node.go +++ b/node.go @@ -1,5 +1,8 @@ package iavl +// NOTE: This file favors int64 as opposed to int for size/counts. +// The Tree on the other hand favors int. This is intentional. + import ( "bytes" "fmt" @@ -17,7 +20,7 @@ type Node struct { value []byte version int64 height int8 - size int + size int64 hash []byte leftHash []byte leftNode *Node @@ -51,11 +54,11 @@ func MakeNode(buf []byte) (node *Node, err error) { n := 1 // Keeps track of bytes read. buf = buf[n:] - node.size, n, err = wire.GetVarint(buf) - if err != nil { - return nil, err - } - buf = buf[n:] + node.size = wire.GetInt64(buf) + buf = buf[8:] + + node.version = wire.GetInt64(buf) + buf = buf[8:] node.key, n, err = wire.GetByteSlice(buf) if err != nil { @@ -63,9 +66,6 @@ func MakeNode(buf []byte) (node *Node, err error) { } buf = buf[n:] - node.version = wire.GetInt64(buf) - buf = buf[8:] - // Read node body. if node.isLeaf() { @@ -100,14 +100,14 @@ func (node *Node) String() string { } // clone creates a shallow copy of a node with its hash set to nil. -func (node *Node) clone() *Node { +func (node *Node) clone(version int64) *Node { if node.isLeaf() { cmn.PanicSanity("Attempt to copy a leaf node") } return &Node{ key: node.key, height: node.height, - version: node.version, + version: version, size: node.size, hash: nil, leftHash: node.leftHash, @@ -138,7 +138,7 @@ func (node *Node) has(t *Tree, key []byte) (has bool) { } // Get a key under the node. -func (node *Node) get(t *Tree, key []byte) (index int, value []byte) { +func (node *Node) get(t *Tree, key []byte) (index int64, value []byte) { if node.isLeaf() { switch bytes.Compare(node.key, key) { case -1: @@ -160,7 +160,7 @@ func (node *Node) get(t *Tree, key []byte) (index int, value []byte) { } } -func (node *Node) getByIndex(t *Tree, index int) (key []byte, value []byte) { +func (node *Node) getByIndex(t *Tree, index int64) (key []byte, value []byte) { if node.isLeaf() { if index == 0 { return node.key, node.value @@ -199,7 +199,7 @@ func (node *Node) _hash() []byte { // Hash the node and its descendants recursively. This usually mutates all // descendant nodes. Returns the node hash and number of nodes hashed. -func (node *Node) hashWithCount() ([]byte, int) { +func (node *Node) hashWithCount() ([]byte, int64) { if node.hash != nil { return node.hash, 0 } @@ -220,14 +220,14 @@ func (node *Node) hashWithCount() ([]byte, int) { // child hashes to be already set. func (node *Node) writeHashBytes(w io.Writer) (n int, err error) { wire.WriteInt8(node.height, w, &n, &err) - wire.WriteVarint(node.size, w, &n, &err) + wire.WriteInt64(node.size, w, &n, &err) + wire.WriteInt64(node.version, w, &n, &err) // Key is not written for inner nodes, unlike writeBytes. if node.isLeaf() { wire.WriteByteSlice(node.key, w, &n, &err) wire.WriteByteSlice(node.value, w, &n, &err) - wire.WriteInt64(node.version, w, &n, &err) } else { if node.leftHash == nil || node.rightHash == nil { cmn.PanicSanity("Found an empty child hash") @@ -240,7 +240,7 @@ func (node *Node) writeHashBytes(w io.Writer) (n int, err error) { // Writes the node's hash to the given io.Writer. // This function has the side-effect of calling hashWithCount. -func (node *Node) writeHashBytesRecursively(w io.Writer) (n int, hashCount int, err error) { +func (node *Node) writeHashBytesRecursively(w io.Writer) (n int, hashCount int64, err error) { if node.leftNode != nil { leftHash, leftCount := node.leftNode.hashWithCount() node.leftHash = leftHash @@ -259,11 +259,11 @@ func (node *Node) writeHashBytesRecursively(w io.Writer) (n int, hashCount int, // Writes the node as a serialized byte slice to the supplied io.Writer. func (node *Node) writeBytes(w io.Writer) (n int, err error) { wire.WriteInt8(node.height, w, &n, &err) - wire.WriteVarint(node.size, w, &n, &err) + wire.WriteInt64(node.size, w, &n, &err) + wire.WriteInt64(node.version, w, &n, &err) // Unlike writeHashBytes, key is written for inner nodes. wire.WriteByteSlice(node.key, w, &n, &err) - wire.WriteInt64(node.version, w, &n, &err) if node.isLeaf() { wire.WriteByteSlice(node.value, w, &n, &err) @@ -311,7 +311,7 @@ func (node *Node) set(t *Tree, key []byte, value []byte) ( } } else { orphaned = append(orphaned, node) - node = node.clone() + node = node.clone(version) if bytes.Compare(key, node.key) < 0 { var leftOrphaned []*Node @@ -341,6 +341,8 @@ func (node *Node) set(t *Tree, key []byte, value []byte) ( func (node *Node) remove(t *Tree, key []byte) ( newHash []byte, newNode *Node, newKey []byte, value []byte, orphaned []*Node, ) { + version := t.version + 1 + if node.isLeaf() { if bytes.Equal(key, node.key) { return nil, nil, nil, node.value, []*Node{node} @@ -362,7 +364,7 @@ func (node *Node) remove(t *Tree, key []byte) ( } orphaned = append(orphaned, node) - newNode := node.clone() + newNode := node.clone(version) newNode.leftHash, newNode.leftNode = newLeftHash, newLeftNode newNode.calcHeightAndSize(t) newNode, balanceOrphaned := newNode.balance(t) @@ -382,7 +384,7 @@ func (node *Node) remove(t *Tree, key []byte) ( } orphaned = append(orphaned, node) - newNode := node.clone() + newNode := node.clone(version) newNode.rightHash, newNode.rightNode = newRightHash, newRightNode if newKey != nil { newNode.key = newKey @@ -410,10 +412,12 @@ func (node *Node) getRightNode(t *Tree) *Node { // Rotate right and return the new node and orphan. func (node *Node) rotateRight(t *Tree) (newNode *Node, orphan *Node) { + version := t.version + 1 + // TODO: optimize balance & rotate. - node = node.clone() + node = node.clone(version) l := node.getLeftNode(t) - _l := l.clone() + _l := l.clone(version) _lrHash, _lrCached := _l.rightHash, _l.rightNode _l.rightHash, _l.rightNode = node.hash, node @@ -427,10 +431,12 @@ func (node *Node) rotateRight(t *Tree) (newNode *Node, orphan *Node) { // Rotate left and return the new node and orphan. func (node *Node) rotateLeft(t *Tree) (newNode *Node, orphan *Node) { + version := t.version + 1 + // TODO: optimize balance & rotate. - node = node.clone() + node = node.clone(version) r := node.getRightNode(t) - _r := r.clone() + _r := r.clone(version) _rlHash, _rlCached := _r.leftHash, _r.leftNode _r.leftHash, _r.leftNode = node.hash, node diff --git a/nodedb.go b/nodedb.go index f6bd7c743..f1ba95778 100644 --- a/nodedb.go +++ b/nodedb.go @@ -15,25 +15,21 @@ import ( var ( // All node keys are prefixed with this. This ensures no collision is // possible with the other keys, and makes them easier to traverse. - nodesPrefix = "n/" - nodesKeyFmt = "n/%x" + nodePrefix = "n/" + nodeKeyFmt = "n/%x" // Orphans are keyed in the database by their expected lifetime. // The first number represents the *last* version at which the orphan needs // to exist, while the second number represents the *earliest* version at // which it is expected to exist - which starts out by being the version // of the node being orphaned. - orphansPrefix = "o/" - orphansPrefixFmt = "o/%d/" // o// - orphansKeyFmt = "o/%d/%d/%x" // o/// - - // These keys are used for the orphan reverse-lookups by node hash. - orphansIndexPrefix = "O/" - orphansIndexKeyFmt = "O/%x" + orphanPrefix = "o/" + orphanPrefixFmt = "o/%d/" // o// + orphanKeyFmt = "o/%d/%d/%x" // o/// // r/ - rootsPrefix = "r/" - rootsPrefixFmt = "r/%d" + rootPrefix = "r/" + rootPrefixFmt = "r/%d" ) type nodeDB struct { @@ -110,6 +106,7 @@ func (ndb *nodeDB) SaveNode(node *Node) { cmn.PanicCrisis(err) } ndb.batch.Set(ndb.nodeKey(node.hash), buf.Bytes()) + debug("BATCH SAVE %X %p\n", node.hash, node) node.persisted = true ndb.cacheNode(node) @@ -129,26 +126,19 @@ func (ndb *nodeDB) Has(hash []byte) bool { return ndb.db.Get(key) != nil } -// SaveBranch saves the given node and all of its descendants. For each node -// about to be saved, the supplied callback is called and the returned node is -// is saved. You may pass nil as the callback as a pass-through. -// -// Note that this function clears leftNode/rigthNode recursively and calls -// hashWithCount on the given node. -func (ndb *nodeDB) SaveBranch(node *Node, cb func(*Node)) []byte { +// SaveBranch saves the given node and all of its descendants. NOTE: This +// function clears leftNode/rigthNode recursively and calls hashWithCount on +// the given node. +func (ndb *nodeDB) SaveBranch(node *Node) []byte { if node.persisted { return node.hash } if node.leftNode != nil { - node.leftHash = ndb.SaveBranch(node.leftNode, cb) + node.leftHash = ndb.SaveBranch(node.leftNode) } if node.rightNode != nil { - node.rightHash = ndb.SaveBranch(node.rightNode, cb) - } - - if cb != nil { - cb(node) + node.rightHash = ndb.SaveBranch(node.rightNode) } node._hash() @@ -169,20 +159,9 @@ func (ndb *nodeDB) DeleteVersion(version int64) { ndb.deleteRoot(version) } -// Unorphan deletes the orphan entry from disk, but not the node it points to. -func (ndb *nodeDB) Unorphan(hash []byte) { - ndb.mtx.Lock() - defer ndb.mtx.Unlock() - - indexKey := ndb.orphanIndexKey(hash) - - if orphansKey := ndb.db.Get(indexKey); len(orphansKey) > 0 { - ndb.batch.Delete(orphansKey) - ndb.batch.Delete(indexKey) - } -} - // Saves orphaned nodes to disk under a special prefix. +// version: the new version being saved. +// orphans: the orphan nodes created since version-1 func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { ndb.mtx.Lock() defer ndb.mtx.Unlock() @@ -190,6 +169,7 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { toVersion := ndb.getPreviousVersion(version) for hash, fromVersion := range orphans { + debug("SAVEORPHAN %v-%v %X\n", fromVersion, toVersion, hash) ndb.saveOrphan([]byte(hash), fromVersion, toVersion) } } @@ -197,14 +177,10 @@ func (ndb *nodeDB) SaveOrphans(version int64, orphans map[string]int64) { // Saves a single orphan to disk. func (ndb *nodeDB) saveOrphan(hash []byte, fromVersion, toVersion int64) { if fromVersion > toVersion { - cmn.PanicSanity("Orphan expires before it comes alive") + panic(fmt.Sprintf("Orphan expires before it comes alive. %d > %d", fromVersion, toVersion)) } key := ndb.orphanKey(fromVersion, toVersion, hash) ndb.batch.Set(key, hash) - - // Set reverse-lookup index. - indexKey := ndb.orphanIndexKey(hash) - ndb.batch.Set(indexKey, key) } // deleteOrphans deletes orphaned nodes from disk, and the associated orphan @@ -217,13 +193,12 @@ func (ndb *nodeDB) deleteOrphans(version int64) { ndb.traverseOrphansVersion(version, func(key, hash []byte) { var fromVersion, toVersion int64 - // See comment on `orphansKeyFmt`. Note that here, `version` and + // See comment on `orphanKeyFmt`. Note that here, `version` and // `toVersion` are always equal. - fmt.Sscanf(string(key), orphansKeyFmt, &toVersion, &fromVersion) + fmt.Sscanf(string(key), orphanKeyFmt, &toVersion, &fromVersion) // Delete orphan key and reverse-lookup key. ndb.batch.Delete(key) - ndb.batch.Delete(ndb.orphanIndexKey(hash)) // If there is no predecessor, or the predecessor is earlier than the // beginning of the lifetime (ie: negative lifetime), or the lifetime @@ -231,28 +206,26 @@ func (ndb *nodeDB) deleteOrphans(version int64) { // can delete the orphan. Otherwise, we shorten its lifetime, by // moving its endpoint to the previous version. if predecessor < fromVersion || fromVersion == toVersion { + debug("DELETE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) ndb.batch.Delete(ndb.nodeKey(hash)) ndb.uncacheNode(hash) } else { + debug("MOVE predecessor:%v fromVersion:%v toVersion:%v %X\n", predecessor, fromVersion, toVersion, hash) ndb.saveOrphan(hash, fromVersion, predecessor) } }) } func (ndb *nodeDB) nodeKey(hash []byte) []byte { - return []byte(fmt.Sprintf(nodesKeyFmt, hash)) -} - -func (ndb *nodeDB) orphanIndexKey(hash []byte) []byte { - return []byte(fmt.Sprintf(orphansIndexKeyFmt, hash)) + return []byte(fmt.Sprintf(nodeKeyFmt, hash)) } func (ndb *nodeDB) orphanKey(fromVersion, toVersion int64, hash []byte) []byte { - return []byte(fmt.Sprintf(orphansKeyFmt, toVersion, fromVersion, hash)) + return []byte(fmt.Sprintf(orphanKeyFmt, toVersion, fromVersion, hash)) } func (ndb *nodeDB) rootKey(version int64) []byte { - return []byte(fmt.Sprintf(rootsPrefixFmt, version)) + return []byte(fmt.Sprintf(rootPrefixFmt, version)) } func (ndb *nodeDB) getLatestVersion() int64 { @@ -264,9 +237,9 @@ func (ndb *nodeDB) getLatestVersion() int64 { func (ndb *nodeDB) getVersions() map[int64][]byte { if len(ndb.versionCache) == 0 { - ndb.traversePrefix([]byte(rootsPrefix), func(k, hash []byte) { + ndb.traversePrefix([]byte(rootPrefix), func(k, hash []byte) { var version int64 - fmt.Sscanf(string(k), rootsPrefixFmt, &version) + fmt.Sscanf(string(k), rootPrefixFmt, &version) ndb.cacheVersion(version, hash) }) } @@ -293,23 +266,22 @@ func (ndb *nodeDB) getPreviousVersion(version int64) int64 { // deleteRoot deletes the root entry from disk, but not the node it points to. func (ndb *nodeDB) deleteRoot(version int64) { - key := ndb.rootKey(version) - ndb.batch.Delete(key) - - delete(ndb.versionCache, version) - if version == ndb.getLatestVersion() { cmn.PanicSanity("Tried to delete latest version") } + + key := ndb.rootKey(version) + ndb.batch.Delete(key) + delete(ndb.versionCache, version) } func (ndb *nodeDB) traverseOrphans(fn func(k, v []byte)) { - ndb.traversePrefix([]byte(orphansPrefix), fn) + ndb.traversePrefix([]byte(orphanPrefix), fn) } // Traverse orphans ending at a certain version. func (ndb *nodeDB) traverseOrphansVersion(version int64, fn func(k, v []byte)) { - prefix := fmt.Sprintf(orphansPrefixFmt, version) + prefix := fmt.Sprintf(orphanPrefixFmt, version) ndb.traversePrefix([]byte(prefix), fn) } @@ -371,9 +343,9 @@ func (ndb *nodeDB) Commit() { func (ndb *nodeDB) getRoots() (map[int64][]byte, error) { roots := map[int64][]byte{} - ndb.traversePrefix([]byte(rootsPrefix), func(k, v []byte) { + ndb.traversePrefix([]byte(rootPrefix), func(k, v []byte) { var version int64 - fmt.Sscanf(string(k), rootsPrefixFmt, &version) + fmt.Sscanf(string(k), rootPrefixFmt, &version) roots[version] = v }) return roots, nil @@ -397,7 +369,7 @@ func (ndb *nodeDB) saveRoot(hash []byte, version int64) error { ndb.mtx.Lock() defer ndb.mtx.Unlock() - if version <= ndb.getLatestVersion() { + if version != ndb.getLatestVersion()+1 { return errors.New("can't save root with lower or equal version than latest") } @@ -457,12 +429,12 @@ func (ndb *nodeDB) size() int { func (ndb *nodeDB) traverseNodes(fn func(hash []byte, node *Node)) { nodes := []*Node{} - ndb.traversePrefix([]byte(nodesPrefix), func(key, value []byte) { + ndb.traversePrefix([]byte(nodePrefix), func(key, value []byte) { node, err := MakeNode(value) if err != nil { cmn.PanicSanity("Couldn't decode node from database") } - fmt.Sscanf(string(key), nodesKeyFmt, &node.hash) + fmt.Sscanf(string(key), nodeKeyFmt, &node.hash) nodes = append(nodes, node) }) @@ -479,7 +451,7 @@ func (ndb *nodeDB) String() string { var str string index := 0 - ndb.traversePrefix([]byte(rootsPrefix), func(key, value []byte) { + ndb.traversePrefix([]byte(rootPrefix), func(key, value []byte) { str += fmt.Sprintf("%s: %x\n", string(key), value) }) str += "\n" @@ -489,20 +461,15 @@ func (ndb *nodeDB) String() string { }) str += "\n" - ndb.traversePrefix([]byte(orphansIndexPrefix), func(key, value []byte) { - str += fmt.Sprintf("%s: %s\n", string(key), value) - }) - str += "\n" - ndb.traverseNodes(func(hash []byte, node *Node) { if len(hash) == 0 { str += fmt.Sprintf("\n") } else if node == nil { - str += fmt.Sprintf("%s%40x: \n", nodesPrefix, hash) + str += fmt.Sprintf("%s%40x: \n", nodePrefix, hash) } else if node.value == nil && node.height > 0 { - str += fmt.Sprintf("%s%40x: %s %-16s h=%d version=%d\n", nodesPrefix, hash, node.key, "", node.height, node.version) + str += fmt.Sprintf("%s%40x: %s %-16s h=%d version=%d\n", nodePrefix, hash, node.key, "", node.height, node.version) } else { - str += fmt.Sprintf("%s%40x: %s = %-16s h=%d version=%d\n", nodesPrefix, hash, node.key, node.value, node.height, node.version) + str += fmt.Sprintf("%s%40x: %s = %-16s h=%d version=%d\n", nodePrefix, hash, node.key, node.value, node.height, node.version) } index++ }) diff --git a/orphaning_tree.go b/orphaning_tree.go index 092ba35c2..15d7d7cb3 100644 --- a/orphaning_tree.go +++ b/orphaning_tree.go @@ -1,6 +1,8 @@ package iavl import ( + "fmt" + cmn "github.com/tendermint/tmlibs/common" ) @@ -36,34 +38,26 @@ func (tree *orphaningTree) Remove(key []byte) ([]byte, bool) { return val, removed } -// Unorphan undoes the orphaning of a node, removing the orphan entry on disk -// if necessary. -func (tree *orphaningTree) unorphan(hash []byte) { - tree.deleteOrphan(hash) - tree.ndb.Unorphan(hash) -} - // SaveAs saves the underlying Tree and assigns it a new version. // Saves orphans too. func (tree *orphaningTree) SaveAs(version int64) { + if version != tree.version+1 { + panic(fmt.Sprintf("Expected to save version %d but tried to save %d", tree.version+1, version)) + } if tree.root == nil { // There can still be orphans, for example if the root is the node being // removed. - tree.ndb.SaveOrphans(tree.version, tree.orphans) + tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveEmptyRoot(version) } else { - // Save the current tree. For each saved node, we delete any existing - // orphan entries in the previous trees. This is necessary because - // sometimes tree re-balancing causes nodes to be incorrectly marked as - // orphaned, since tree patterns after a re-balance may mirror previous - // tree patterns, with matching hashes. - tree.ndb.SaveBranch(tree.root, func(node *Node) { - tree.unorphan(node._hash()) - }) - tree.ndb.SaveOrphans(tree.version, tree.orphans) + debug("SAVE TREE %v\n", version) + // Save the current tree. + tree.ndb.SaveBranch(tree.root) + tree.ndb.SaveOrphans(version, tree.orphans) tree.ndb.SaveRoot(tree.root, version) } tree.ndb.Commit() + tree.version = version } // Add orphans to the orphan list. Doesn't write to disk. @@ -80,8 +74,8 @@ func (tree *orphaningTree) addOrphans(orphans []*Node) { } } -// Delete an orphan from the orphan list. Doesn't write to disk. -func (tree *orphaningTree) deleteOrphan(hash []byte) (version int64, deleted bool) { +// Drop an orphan from the orphan list. Doesn't write to disk. +func (tree *orphaningTree) dropOrphan(hash []byte) (version int64, dropped bool) { if version, ok := tree.orphans[string(hash)]; ok { delete(tree.orphans, string(hash)) return version, true diff --git a/proof.go b/proof.go index 401c58ea1..7d78333b1 100644 --- a/proof.go +++ b/proof.go @@ -27,14 +27,15 @@ var ( ) type proofInnerNode struct { - Height int8 - Size int - Left []byte - Right []byte + Height int8 + Size int64 + Version int64 + Left []byte + Right []byte } func (n *proofInnerNode) String() string { - return fmt.Sprintf("proofInnerNode[height=%d, %x / %x]", n.Height, n.Left, n.Right) + return fmt.Sprintf("proofInnerNode[height=%d, ver=%d %x / %x]", n.Height, n.Version, n.Left, n.Right) } func (branch proofInnerNode) Hash(childHash []byte) []byte { @@ -43,7 +44,8 @@ func (branch proofInnerNode) Hash(childHash []byte) []byte { n, err := int(0), error(nil) wire.WriteInt8(branch.Height, buf, &n, &err) - wire.WriteVarint(branch.Size, buf, &n, &err) + wire.WriteInt64(branch.Size, buf, &n, &err) + wire.WriteInt64(branch.Version, buf, &n, &err) if len(branch.Left) == 0 { wire.WriteByteSlice(childHash, buf, &n, &err) @@ -72,10 +74,10 @@ func (leaf proofLeafNode) Hash() []byte { n, err := int(0), error(nil) wire.WriteInt8(0, buf, &n, &err) - wire.WriteVarint(1, buf, &n, &err) + wire.WriteInt64(1, buf, &n, &err) + wire.WriteInt64(leaf.Version, buf, &n, &err) wire.WriteByteSlice(leaf.KeyBytes, buf, &n, &err) wire.WriteByteSlice(leaf.ValueBytes, buf, &n, &err) - wire.WriteInt64(leaf.Version, buf, &n, &err) if err != nil { cmn.PanicCrisis(cmn.Fmt("Failed to hash proofLeafNode: %v", err)) @@ -111,10 +113,11 @@ func (node *Node) _pathToKey(t *Tree, key []byte, path *PathToKey) (*Node, error return nil, err } else { branch := proofInnerNode{ - Height: node.height, - Size: node.size, - Left: nil, - Right: node.getRightNode(t).hash, + Height: node.height, + Size: node.size, + Version: node.version, + Left: nil, + Right: node.getRightNode(t).hash, } path.InnerNodes = append(path.InnerNodes, branch) return n, nil @@ -125,10 +128,11 @@ func (node *Node) _pathToKey(t *Tree, key []byte, path *PathToKey) (*Node, error return nil, err } else { branch := proofInnerNode{ - Height: node.height, - Size: node.size, - Left: node.getLeftNode(t).hash, - Right: nil, + Height: node.height, + Size: node.size, + Version: node.version, + Left: node.getLeftNode(t).hash, + Right: nil, } path.InnerNodes = append(path.InnerNodes, branch) return n, nil @@ -137,7 +141,7 @@ func (node *Node) _pathToKey(t *Tree, key []byte, path *PathToKey) (*Node, error func (t *Tree) constructKeyAbsentProof(key []byte, proof *KeyAbsentProof) error { // Get the index of the first key greater than the requested key, if the key doesn't exist. - idx, val := t.Get(key) + idx, val := t.Get64(key) if val != nil { return errors.Errorf("couldn't construct non-existence proof: key 0x%x exists", key) } @@ -147,10 +151,10 @@ func (t *Tree) constructKeyAbsentProof(key []byte, proof *KeyAbsentProof) error rkey, rval []byte ) if idx > 0 { - lkey, lval = t.GetByIndex(idx - 1) + lkey, lval = t.GetByIndex64(idx - 1) } - if idx <= t.Size()-1 { - rkey, rval = t.GetByIndex(idx) + if idx <= t.Size64()-1 { + rkey, rval = t.GetByIndex64(idx) } if lkey == nil && rkey == nil { diff --git a/proof_range.go b/proof_range.go index 441921b8e..8b1aeb568 100644 --- a/proof_range.go +++ b/proof_range.go @@ -298,8 +298,8 @@ func (t *Tree) getRangeWithProof(keyStart, keyEnd []byte, limit int) ( if needsLeft { // Find index of first key to the left, and include proof if it isn't the // leftmost key. - if idx, _ := t.Get(rangeStart); idx > 0 { - lkey, lval := t.GetByIndex(idx - 1) + if idx, _ := t.Get64(rangeStart); idx > 0 { + lkey, lval := t.GetByIndex64(idx - 1) path, node, _ := t.root.pathToKey(t, lkey) rangeProof.Left = &pathWithNode{ Path: path, @@ -314,8 +314,8 @@ func (t *Tree) getRangeWithProof(keyStart, keyEnd []byte, limit int) ( if needsRight { // Find index of first key to the right, and include proof if it isn't the // rightmost key. - if idx, _ := t.Get(rangeEnd); idx <= t.Size()-1 { - rkey, rval := t.GetByIndex(idx) + if idx, _ := t.Get64(rangeEnd); idx <= t.Size64()-1 { + rkey, rval := t.GetByIndex64(idx) path, node, _ := t.root.pathToKey(t, rkey) rangeProof.Right = &pathWithNode{ Path: path, @@ -349,8 +349,8 @@ func (t *Tree) getFirstInRangeWithProof(keyStart, keyEnd []byte) ( } if !bytes.Equal(key, keyStart) { - if idx, _ := t.Get(keyStart); idx-1 >= 0 && idx-1 <= t.Size()-1 { - k, v := t.GetByIndex(idx - 1) + if idx, _ := t.Get64(keyStart); idx-1 >= 0 && idx-1 <= t.Size64()-1 { + k, v := t.GetByIndex64(idx - 1) path, node, _ := t.root.pathToKey(t, k) proof.Left = &pathWithNode{ Path: path, @@ -360,8 +360,8 @@ func (t *Tree) getFirstInRangeWithProof(keyStart, keyEnd []byte) ( } if !bytes.Equal(key, keyEnd) { - if idx, val := t.Get(keyEnd); idx <= t.Size()-1 && val == nil { - k, v := t.GetByIndex(idx) + if idx, val := t.Get64(keyEnd); idx <= t.Size64()-1 && val == nil { + k, v := t.GetByIndex64(idx) path, node, _ := t.root.pathToKey(t, k) proof.Right = &pathWithNode{ Path: path, @@ -396,8 +396,8 @@ func (t *Tree) getLastInRangeWithProof(keyStart, keyEnd []byte) ( } if !bytes.Equal(key, keyEnd) { - if idx, _ := t.Get(keyEnd); idx <= t.Size()-1 { - k, v := t.GetByIndex(idx) + if idx, _ := t.Get64(keyEnd); idx <= t.Size64()-1 { + k, v := t.GetByIndex64(idx) path, node, _ := t.root.pathToKey(t, k) proof.Right = &pathWithNode{ Path: path, @@ -407,8 +407,8 @@ func (t *Tree) getLastInRangeWithProof(keyStart, keyEnd []byte) ( } if !bytes.Equal(key, keyStart) { - if idx, _ := t.Get(keyStart); idx-1 >= 0 && idx-1 <= t.Size()-1 { - k, v := t.GetByIndex(idx - 1) + if idx, _ := t.Get64(keyStart); idx-1 >= 0 && idx-1 <= t.Size64()-1 { + k, v := t.GetByIndex64(idx - 1) path, node, _ := t.root.pathToKey(t, k) proof.Left = &pathWithNode{ Path: path, diff --git a/tree.go b/tree.go index f7f267845..cd7452e90 100644 --- a/tree.go +++ b/tree.go @@ -43,14 +43,31 @@ func (t *Tree) String() string { // Size returns the number of leaf nodes in the tree. func (t *Tree) Size() int { + return int(t.Size64()) +} + +func (t *Tree) Size64() int64 { if t.root == nil { return 0 } return t.root.size } +// Version returns the version of the tree. +func (t *Tree) Version() int { + return int(t.Version64()) +} + +func (t *Tree) Version64() int64 { + return t.version +} + // Height returns the height of the tree. -func (t *Tree) Height() int8 { +func (t *Tree) Height() int { + return int(t.Height8()) +} + +func (t *Tree) Height8() int8 { if t.root == nil { return 0 } @@ -94,7 +111,7 @@ func (t *Tree) Hash() []byte { } // hashWithCount returns the root hash and hash count. -func (t *Tree) hashWithCount() ([]byte, int) { +func (t *Tree) hashWithCount() ([]byte, int64) { if t.root == nil { return nil, 0 } @@ -104,6 +121,11 @@ func (t *Tree) hashWithCount() ([]byte, int) { // Get returns the index and value of the specified key if it exists, or nil // and the next index, if it doesn't. func (t *Tree) Get(key []byte) (index int, value []byte) { + index64, value := t.Get64(key) + return int(index64), value +} + +func (t *Tree) Get64(key []byte) (index int64, value []byte) { if t.root == nil { return 0, nil } @@ -112,6 +134,10 @@ func (t *Tree) Get(key []byte) (index int, value []byte) { // GetByIndex gets the key and value at the specified index. func (t *Tree) GetByIndex(index int) (key []byte, value []byte) { + return t.GetByIndex64(int64(index)) +} + +func (t *Tree) GetByIndex64(index int64) (key []byte, value []byte) { if t.root == nil { return nil, nil } @@ -233,12 +259,13 @@ func (tree *Tree) clone() *Tree { // Load the tree from disk, from the given root hash, including all orphans. // Used internally by VersionedTree. -func (tree *Tree) load(root []byte) { +func (tree *Tree) load(version int64, root []byte) { if len(root) == 0 { tree.root = nil return } tree.root = tree.ndb.GetNode(root) + tree.version = version } // nodeSize is like Size, but includes inner nodes too. diff --git a/tree_test.go b/tree_test.go index 0047b73bb..3b37aca64 100644 --- a/tree_test.go +++ b/tree_test.go @@ -224,7 +224,7 @@ func TestVersionedEmptyTree(t *testing.T) { require.EqualValues(4, v) require.NoError(err) - require.EqualValues(4, tree.LatestVersion()) + require.EqualValues(4, tree.Version()) require.True(tree.VersionExists(1)) require.True(tree.VersionExists(3)) @@ -303,7 +303,7 @@ func TestVersionedTree(t *testing.T) { require.NoError(tree.Load()) require.Len(tree.versions, 2, "wrong number of versions") - require.EqualValues(v2, tree.LatestVersion()) + require.EqualValues(v2, tree.Version()) // -----1----- // key1 = val0 @@ -613,14 +613,14 @@ func TestVersionedTreeSaveAndLoad(t *testing.T) { preHash := tree.Hash() require.NotNil(preHash) - require.Equal(int64(6), tree.LatestVersion()) + require.Equal(6, tree.Version()) // Reload the tree, to test that roots and orphans are properly loaded. ntree := NewVersionedTree(0, d) ntree.Load() require.False(ntree.IsEmpty()) - require.Equal(int64(6), ntree.LatestVersion()) + require.Equal(6, ntree.Version()) postHash := ntree.Hash() require.Equal(preHash, postHash) @@ -1042,7 +1042,7 @@ func TestCopyValueSemantics(t *testing.T) { require.Equal([]byte("v2"), val) } -func TestResetToLatest(t *testing.T) { +func TestRollback(t *testing.T) { require := require.New(t) tree := NewVersionedTree(0, db.NewMemDB()) @@ -1053,7 +1053,7 @@ func TestResetToLatest(t *testing.T) { tree.Set([]byte("r"), []byte("v")) tree.Set([]byte("s"), []byte("v")) - tree.ResetToLatest() + tree.Rollback() tree.Set([]byte("t"), []byte("v")) @@ -1071,42 +1071,6 @@ func TestResetToLatest(t *testing.T) { require.Equal([]byte("v"), val) } -func TestLoadVersion(t *testing.T) { - require := require.New(t) - - d := db.NewMemDB() - tree := NewVersionedTree(0, d) - - tree.Set([]byte("k"), []byte("v")) - tree.SaveVersion() - - tree.Set([]byte("r"), []byte("v")) - tree.SaveVersion() - - tree.Set([]byte("s"), []byte("v")) - tree.SaveVersion() - - tree.Set([]byte("k"), []byte("u")) - tree.SaveVersion() - - tree = NewVersionedTree(0, d) - tree.LoadVersion(2) - - require.EqualValues(2, tree.LatestVersion()) - require.Len(tree.versions, 2) - - _, val := tree.Get([]byte("r")) - require.Equal([]byte("v"), val) - - _, val = tree.Get([]byte("k")) - require.Equal([]byte("v"), val) - - _, val = tree.Get([]byte("s")) - require.Nil(val) - - require.Error(tree.LoadVersion(5)) -} - //////////////////////////// BENCHMARKS /////////////////////////////////////// func BenchmarkTreeLoadAndDelete(b *testing.B) { diff --git a/versioned_tree.go b/versioned_tree.go index 8d85ae62e..d567a5eaa 100644 --- a/versioned_tree.go +++ b/versioned_tree.go @@ -28,11 +28,6 @@ func NewVersionedTree(cacheSize int, db dbm.DB) *VersionedTree { } } -// LatestVersion returns the latest saved version of the tree. -func (tree *VersionedTree) LatestVersion() int64 { - return tree.orphaningTree.version -} - // IsEmpty returns whether or not the tree has any keys. Only trees that are // not empty can be saved. func (tree *VersionedTree) IsEmpty() bool { @@ -74,35 +69,6 @@ func (tree *VersionedTree) Remove(key []byte) ([]byte, bool) { return tree.orphaningTree.Remove(key) } -// LoadVersion loads the given version and all versions prior. -func (tree *VersionedTree) LoadVersion(version int64) error { - roots, err := tree.ndb.getRoots() - if err != nil { - return err - } - if len(roots) == 0 { - return nil - } - if _, ok := roots[version]; !ok { - return errors.WithStack(ErrVersionDoesNotExist) - } - - // Load all roots from the database up to the given version. - for v, root := range roots { - if v > version { - continue - } - t := &Tree{ndb: tree.ndb, version: v} - t.load(root) - - tree.versions[v] = t - } - tree.version = version - tree.ResetToLatest() - - return nil -} - // Load a versioned tree from disk. All tree versions are loaded automatically. func (tree *VersionedTree) Load() error { roots, err := tree.ndb.getRoots() @@ -117,7 +83,7 @@ func (tree *VersionedTree) Load() error { latestVersion := int64(0) for version, root := range roots { t := &Tree{ndb: tree.ndb, version: version} - t.load(root) + t.load(version, root) tree.versions[version] = t @@ -133,9 +99,9 @@ func (tree *VersionedTree) Load() error { return nil } -// ResetToLatest resets the working tree to the latest saved version, discarding +// Rollback resets the working tree to the latest saved version, discarding // any unsaved modifications. -func (tree *VersionedTree) ResetToLatest() { +func (tree *VersionedTree) Rollback() { if tree.version > 0 { tree.orphaningTree = newOrphaningTree( tree.versions[tree.version].clone(), @@ -164,18 +130,14 @@ func (tree *VersionedTree) SaveVersion() ([]byte, int64, error) { return nil, version, errors.Errorf("version %d was already saved", version) } - tree.version = version // Sets the inner *Tree version. + // Persist version and stash to .versions. + tree.orphaningTree.SaveAs(version) tree.versions[version] = tree.orphaningTree.Tree - tree.orphaningTree.SaveAs(version) - tree.orphaningTree = newOrphaningTree( - tree.versions[version].clone(), - ) + // Set new working tree. + tree.orphaningTree = newOrphaningTree(tree.orphaningTree.clone()) - if tree.root == nil { - return nil, version, nil - } - return tree.root.hash, version, nil + return tree.Hash(), version, nil } // DeleteVersion deletes a tree version from disk. The version can then no