From f1117f8c091171b9e2492210e94dd6d32aba04fc Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Mon, 3 Apr 2023 14:16:42 -0400 Subject: [PATCH 1/2] fix traverseNodes --- docs/tree/mutable_tree.md | 4 +- go.mod | 6 +- go.sum | 12 +-- nodedb.go | 151 ++++++++++++++++++++------------------ 4 files changed, 89 insertions(+), 84 deletions(-) diff --git a/docs/tree/mutable_tree.md b/docs/tree/mutable_tree.md index bdea1696d..9ce62eb1c 100644 --- a/docs/tree/mutable_tree.md +++ b/docs/tree/mutable_tree.md @@ -152,9 +152,9 @@ After `RotateRight(node8)`: | |---6 | |---5 4'---| - | |---1 + | |---3 |---2 - |---3 + |---1 Orphaned: 4, 8 ``` diff --git a/go.mod b/go.mod index 98258ce22..b19ed4db7 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.18 require ( github.com/cosmos/cosmos-db v1.0.0-rc.1 github.com/cosmos/ics23/go v0.9.1-0.20221207100636-b1abd8678aab - github.com/emicklei/dot v1.3.1 + github.com/emicklei/dot v1.4.2 github.com/golang/mock v1.6.0 - github.com/golangci/golangci-lint v1.52.0 + github.com/golangci/golangci-lint v1.52.2 github.com/stretchr/testify v1.8.2 golang.org/x/crypto v0.7.0 ) @@ -123,7 +123,7 @@ require ( github.com/mgechev/revive v1.3.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moricho/tparallel v0.3.0 // indirect + github.com/moricho/tparallel v0.3.1 // indirect github.com/nakabonne/nestif v0.3.1 // indirect github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect github.com/nishanths/exhaustive v0.9.5 // indirect diff --git a/go.sum b/go.sum index 1406a1932..b06b2ed33 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= -github.com/emicklei/dot v1.3.1 h1:45fM+Qyq25fvT4jWNZnvJp3bGWl6OdxC1vxJ7+FWXQM= -github.com/emicklei/dot v1.3.1/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= +github.com/emicklei/dot v1.4.2 h1:UbK6gX4yvrpHKlxuUQicwoAis4zl8Dzwit9SnbBAXWw= +github.com/emicklei/dot v1.4.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -285,8 +285,8 @@ github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe h1:6RGUuS7EGotKx6 github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe/go.mod h1:gjqyPShc/m8pEMpk0a3SeagVb0kaqvhscv+i9jI5ZhQ= github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2 h1:amWTbTGqOZ71ruzrdA+Nx5WA3tV1N0goTspwmKCQvBY= github.com/golangci/gofmt v0.0.0-20220901101216-f2edd75033f2/go.mod h1:9wOXstvyDRshQ9LggQuzBCGysxs3b6Uo/1MvYCR2NMs= -github.com/golangci/golangci-lint v1.52.0 h1:T7w3tuF1goz64qGV+ML4MgysSl/yUfA3UZJK92oE48A= -github.com/golangci/golangci-lint v1.52.0/go.mod h1:wlTh+d/oVlgZC2yCe6nlxrxNAnuhEQC0Zdygoh72Uak= +github.com/golangci/golangci-lint v1.52.2 h1:FrPElUUI5rrHXg1mQ7KxI1MXPAw5lBVskiz7U7a8a1A= +github.com/golangci/golangci-lint v1.52.2/go.mod h1:S5fhC5sHM5kE22/HcATKd1XLWQxX+y7mHj8B5H91Q/0= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= @@ -493,8 +493,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/moricho/tparallel v0.3.0 h1:8dDx3S3e+jA+xiQXC7O3dvfRTe/J+FYlTDDW01Y7z/Q= -github.com/moricho/tparallel v0.3.0/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI= +github.com/moricho/tparallel v0.3.1 h1:fQKD4U1wRMAYNngDonW5XupoB/ZGJHdpzrWqgyg9krA= +github.com/moricho/tparallel v0.3.1/go.mod h1:leENX2cUv7Sv2qDgdi0D0fCftN8fRC67Bcn8pqzeYNI= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= diff --git a/nodedb.go b/nodedb.go index 54933c2e6..2065a8907 100644 --- a/nodedb.go +++ b/nodedb.go @@ -511,6 +511,15 @@ func (ndb *nodeDB) HasVersion(version int64) (bool, error) { return ndb.db.Has(nodeKeyFormat.Key(version, []byte{1})) } +func isReferenceToRoot(bz []byte) bool { + if bz[0] == nodeKeyFormat.Prefix()[0] { + if len(bz) == 13 { + return true + } + } + return false +} + // GetRoot gets the nodeKey of the root for the specific version. func (ndb *nodeDB) GetRoot(version int64) (*NodeKey, error) { val, err := ndb.db.Get(nodeKeyFormat.Key(version, []byte{1})) @@ -523,7 +532,7 @@ func (ndb *nodeDB) GetRoot(version int64) (*NodeKey, error) { if len(val) == 0 { // empty root return nil, nil } - if val[0] == nodeKeyFormat.Prefix()[0] { // point to the prev root + if isReferenceToRoot(val) { // point to the prev root var ( version int64 nonce int32 @@ -702,6 +711,56 @@ func (ndb *nodeDB) traverseOrphans(version int64, fn func(*Node) error) error { return nil } +// traverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it. +// endVersion is exclusive, set to `math.MaxInt64` to cover the latest version. +func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error { + firstVersion, err := ndb.getFirstVersion() + if err != nil { + return err + } + if startVersion < firstVersion { + startVersion = firstVersion + } + latestVersion, err := ndb.getLatestVersion() + if err != nil { + return err + } + if endVersion > latestVersion { + endVersion = latestVersion + } + + prevVersion := startVersion - 1 + prevRoot, err := ndb.GetRoot(prevVersion) + if err != nil && err != ErrVersionDoesNotExist { + return err + } + + for version := startVersion; version <= endVersion; version++ { + root, err := ndb.GetRoot(version) + if err != nil { + return err + } + + var changeSet ChangeSet + receiveKVPair := func(pair *KVPair) error { + changeSet.Pairs = append(changeSet.Pairs, *pair) + return nil + } + + if err := ndb.extractStateChanges(prevVersion, prevRoot, root, receiveKVPair); err != nil { + return err + } + + if err := fn(version, &changeSet); err != nil { + return err + } + prevVersion = version + prevRoot = root + } + + return nil +} + // Utility and test functions func (ndb *nodeDB) leafNodes() ([]*Node, error) { @@ -768,32 +827,28 @@ func (ndb *nodeDB) size() int { } func (ndb *nodeDB) traverseNodes(fn func(node *Node) error) error { - ndb.resetLatestVersion(0) - latest, err := ndb.getLatestVersion() - if err != nil { - return err - } - nodes := []*Node{} - for version := int64(1); version <= latest; version++ { - if err := ndb.traverseRange(nodeKeyFormat.Key(version), nodeKeyFormat.Key(version+1), func(key, value []byte) error { - var ( - version int64 - nonce int32 - ) - nodeKeyFormat.Scan(key, &version, &nonce) - node, err := MakeNode(&NodeKey{ - version: version, - nonce: nonce, - }, value) - if err != nil { - return err - } - nodes = append(nodes, node) + + if err := ndb.traversePrefix(nodeKeyFormat.Key(), func(key, value []byte) error { + if isReferenceToRoot(value) { return nil - }); err != nil { + } + var ( + version int64 + nonce int32 + ) + nodeKeyFormat.Scan(key, &version, &nonce) + node, err := MakeNode(&NodeKey{ + version: version, + nonce: nonce, + }, value) + if err != nil { return err } + nodes = append(nodes, node) + return nil + }); err != nil { + return err } sort.Slice(nodes, func(i, j int) bool { @@ -808,56 +863,6 @@ func (ndb *nodeDB) traverseNodes(fn func(node *Node) error) error { return nil } -// traverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it. -// endVersion is exclusive, set to `math.MaxInt64` to cover the latest version. -func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error { - firstVersion, err := ndb.getFirstVersion() - if err != nil { - return err - } - if startVersion < firstVersion { - startVersion = firstVersion - } - latestVersion, err := ndb.getLatestVersion() - if err != nil { - return err - } - if endVersion > latestVersion { - endVersion = latestVersion - } - - prevVersion := startVersion - 1 - prevRoot, err := ndb.GetRoot(prevVersion) - if err != nil && err != ErrVersionDoesNotExist { - return err - } - - for version := startVersion; version <= endVersion; version++ { - root, err := ndb.GetRoot(version) - if err != nil { - return err - } - - var changeSet ChangeSet - receiveKVPair := func(pair *KVPair) error { - changeSet.Pairs = append(changeSet.Pairs, *pair) - return nil - } - - if err := ndb.extractStateChanges(prevVersion, prevRoot, root, receiveKVPair); err != nil { - return err - } - - if err := fn(version, &changeSet); err != nil { - return err - } - prevVersion = version - prevRoot = root - } - - return nil -} - func (ndb *nodeDB) String() (string, error) { buf := bufPool.Get().(*bytes.Buffer) defer bufPool.Put(buf) From fea9601d0568d9acc92931a36b63809b5065ddfd Mon Sep 17 00:00:00 2001 From: Cool Developer Date: Mon, 3 Apr 2023 14:56:55 -0400 Subject: [PATCH 2/2] add test --- nodedb_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/nodedb_test.go b/nodedb_test.go index ba33c2a79..381985de8 100644 --- a/nodedb_test.go +++ b/nodedb_test.go @@ -253,6 +253,38 @@ func TestIsFastStorageEnabled_False(t *testing.T) { require.NoError(t, err) } +func TestTraverseNodes(t *testing.T) { + tree, _ := getTestTree(0) + // version 1 + for i := 0; i < 20; i++ { + _, err := tree.Set([]byte{byte(i)}, []byte{byte(i)}) + require.NoError(t, err) + } + _, _, err := tree.SaveVersion() + require.NoError(t, err) + // version 2, no commit + _, _, err = tree.SaveVersion() + require.NoError(t, err) + // version 3 + for i := 20; i < 30; i++ { + _, err := tree.Set([]byte{byte(i)}, []byte{byte(i)}) + require.NoError(t, err) + } + _, _, err = tree.SaveVersion() + require.NoError(t, err) + + count := 0 + err = tree.ndb.traverseNodes(func(node *Node) error { + t.Log(node) + if node.isLeaf() { + count++ + } + return nil + }) + require.NoError(t, err) + require.Equal(t, 30, count) +} + func assertOrphansAndBranches(t *testing.T, ndb *nodeDB, version int64, branches int, orphanKeys [][]byte) { var branchCount, orphanIndex int err := ndb.traverseOrphans(version, func(node *Node) error {