From 76afdeafb9642c0a113582684baa4e7f92962f86 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 12 Dec 2022 18:17:53 +0800 Subject: [PATCH 1/2] feat: speed up rollback command (#636) (cherry picked from commit d4086fe57922ec10dadd92d9bdd63e376a271144) # Conflicts: # nodedb.go --- CHANGELOG.md | 1 + mutable_tree.go | 19 +++++++------------ nodedb.go | 17 ++++++++++++++--- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5d3e246..0cd91992d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ## Unreleased - [#640](https://github.com/cosmos/iavl/pull/640) commit `NodeDB` batch in `LoadVersionForOverwriting`. +- [#636](https://github.com/cosmos/iavl/pull/636) Speed up rollback method: `LoadVersionForOverwriting`. ## 0.19.4 (October 28, 2022) diff --git a/mutable_tree.go b/mutable_tree.go index f704b375f..831a142e4 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -630,17 +630,18 @@ func (tree *MutableTree) LoadVersionForOverwriting(targetVersion int64) (int64, return latestVersion, err } + tree.mtx.Lock() + defer tree.mtx.Unlock() + + tree.ndb.resetLatestVersion(latestVersion) + if !tree.skipFastStorageUpgrade { - if err := tree.enableFastStorageAndCommitLocked(); err != nil { + // it'll repopulates the fast node index because of version mismatch. + if _, err := tree.enableFastStorageAndCommitIfNotEnabled(); err != nil { return latestVersion, err } } - tree.ndb.resetLatestVersion(latestVersion) - - tree.mtx.Lock() - defer tree.mtx.Unlock() - for v := range tree.versions { if v > targetVersion { delete(tree.versions, v) @@ -706,12 +707,6 @@ func (tree *MutableTree) enableFastStorageAndCommitIfNotEnabled() (bool, error) return true, nil } -func (tree *MutableTree) enableFastStorageAndCommitLocked() error { - tree.mtx.Lock() - defer tree.mtx.Unlock() - return tree.enableFastStorageAndCommit() -} - func (tree *MutableTree) enableFastStorageAndCommit() error { var err error diff --git a/nodedb.go b/nodedb.go index 2d4686fe6..f1f06940d 100644 --- a/nodedb.go +++ b/nodedb.go @@ -32,6 +32,7 @@ const ( defaultStorageVersionValue = "1.0.0" fastStorageVersionValue = "1.1.0" fastNodeCacheSize = 100000 + maxVersion = int64(math.MaxInt64) ) var ( @@ -451,7 +452,7 @@ func (ndb *nodeDB) DeleteVersionsFrom(version int64) error { // Next, delete orphans: // - Delete orphan entries *and referred nodes* with fromVersion >= version // - Delete orphan entries with toVersion >= version-1 (since orphans at latest are not orphans) - err = ndb.traverseOrphans(func(key, hash []byte) error { + err = ndb.traverseRange(orphanKeyFormat.Key(version-1), orphanKeyFormat.Key(maxVersion), func(key, hash []byte) error { var fromVersion, toVersion int64 orphanKeyFormat.Scan(key, &toVersion, &fromVersion) @@ -476,7 +477,7 @@ func (ndb *nodeDB) DeleteVersionsFrom(version int64) error { } // Delete the version root entries - err = ndb.traverseRange(rootKeyFormat.Key(version), rootKeyFormat.Key(int64(math.MaxInt64)), func(k, v []byte) error { + err = ndb.traverseRange(rootKeyFormat.Key(version), rootKeyFormat.Key(maxVersion), func(k, v []byte) error { if err = ndb.batch.Delete(k); err != nil { return err } @@ -487,6 +488,7 @@ func (ndb *nodeDB) DeleteVersionsFrom(version int64) error { return err } +<<<<<<< HEAD // Delete fast node entries err = ndb.traverseFastNodes(func(keyWithPrefix, v []byte) error { key := keyWithPrefix[1:] @@ -508,6 +510,9 @@ func (ndb *nodeDB) DeleteVersionsFrom(version int64) error { if err != nil { return err } +======= + // NOTICE: we don't touch fast node indexes here, because it'll be rebuilt later because of version mismatch. +>>>>>>> d4086fe (feat: speed up rollback command (#636)) return nil } @@ -605,6 +610,11 @@ func (ndb *nodeDB) deleteNodesFrom(version int64, hash []byte) error { return err } + if node.version < version { + // We can skip the whole sub-tree since children.version <= parent.version. + return nil + } + if node.leftHash != nil { if err := ndb.deleteNodesFrom(version, node.leftHash); err != nil { return err @@ -722,7 +732,7 @@ func (ndb *nodeDB) rootKey(version int64) []byte { func (ndb *nodeDB) getLatestVersion() (int64, error) { if ndb.latestVersion == 0 { var err error - ndb.latestVersion, err = ndb.getPreviousVersion(1<<63 - 1) + ndb.latestVersion, err = ndb.getPreviousVersion(maxVersion) if err != nil { return 0, err } @@ -786,6 +796,7 @@ func (ndb *nodeDB) traverseOrphans(fn func(keyWithPrefix, v []byte) error) error } // Traverse fast nodes and return error if any, nil otherwise +// nolint: unused func (ndb *nodeDB) traverseFastNodes(fn func(k, v []byte) error) error { return ndb.traversePrefix(fastKeyFormat.Key(), fn) } From 37aa3efb7d4baf280cbd75e03e9e2deb9aac8e54 Mon Sep 17 00:00:00 2001 From: Marko Baricevic Date: Fri, 16 Dec 2022 10:09:09 +0700 Subject: [PATCH 2/2] conflicts --- nodedb.go | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/nodedb.go b/nodedb.go index f1f06940d..d555b9d6b 100644 --- a/nodedb.go +++ b/nodedb.go @@ -488,31 +488,7 @@ func (ndb *nodeDB) DeleteVersionsFrom(version int64) error { return err } -<<<<<<< HEAD - // Delete fast node entries - err = ndb.traverseFastNodes(func(keyWithPrefix, v []byte) error { - key := keyWithPrefix[1:] - fastNode, err := DeserializeFastNode(key, v) - - if err != nil { - return err - } - - if version <= fastNode.versionLastUpdatedAt { - if err = ndb.batch.Delete(keyWithPrefix); err != nil { - return err - } - ndb.fastNodeCache.Remove(key) - } - return nil - }) - - if err != nil { - return err - } -======= // NOTICE: we don't touch fast node indexes here, because it'll be rebuilt later because of version mismatch. ->>>>>>> d4086fe (feat: speed up rollback command (#636)) return nil } @@ -1019,6 +995,7 @@ func (ndb *nodeDB) orphans() ([][]byte, error) { // Not efficient. // NOTE: DB cannot implement Size() because // mutations are not always synchronous. +// //nolint:unused func (ndb *nodeDB) size() int { size := 0