Problem: dynamic-level-bytes is not enabled for restored application.…

…db (backport: #904) (#905) * Problem: dynamic-level-bytes is not enabled for restored application.db Solution: - enable it on new db, but don't override existing db - another implication is, user can manually modify the OPTIONS file to tune the options that's not set by app * remove flagsNumLevels * Update cmd/cronosd/opendb/opendb_rocksdb.go Signed-off-by: yihuang <huang@crypto.com> * deduplicate code * use upstream * fix build --------- Signed-off-by: yihuang <huang@crypto.com>
crypto-org-chain · Mar 3, 2023 · 1eebd48 · 1eebd48
1 parent aab9d81
commit 1eebd48
Show file tree

Hide file tree

Showing 11 changed files with 73 additions and 61 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## UNRELEASED
+
+### Improvements
+
+- [#904](https://github.com/crypto-org-chain/cronos/pull/904) Enable "dynamic-level-bytes" on new `application.db`.
+
 *Feb 15, 2022*
 
 ## v1.0.4

diff --git a/cmd/cronosd/cmd/versiondb.go b/cmd/cronosd/cmd/versiondb.go
@@ -9,6 +9,7 @@ import (
 	"github.com/crypto-org-chain/cronos/app"
 	"github.com/crypto-org-chain/cronos/cmd/cronosd/opendb"
 	versiondbclient "github.com/crypto-org-chain/cronos/versiondb/client"
+	"github.com/linxGnu/grocksdb"
 	"github.com/spf13/cobra"
 )
 
@@ -21,8 +22,10 @@ func ChangeSetCmd() *cobra.Command {
 	sort.Strings(storeNames)
 
 	return versiondbclient.ChangeSetGroupCmd(versiondbclient.Options{
-		DefaultStores:     storeNames,
-		OpenReadOnlyDB:    opendb.OpenReadOnlyDB,
-		AppRocksDBOptions: opendb.NewRocksdbOptions,
+		DefaultStores:  storeNames,
+		OpenReadOnlyDB: opendb.OpenReadOnlyDB,
+		AppRocksDBOptions: func(sstFileWriter bool) *grocksdb.Options {
+			return opendb.NewRocksdbOptions(nil, sstFileWriter)
+		},
 	})
 }
diff --git a/cmd/cronosd/opendb/opendb_rocksdb.go b/cmd/cronosd/opendb/opendb_rocksdb.go
@@ -6,24 +6,18 @@ package opendb
 import (
 	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/linxGnu/grocksdb"
 	dbm "github.com/tendermint/tm-db"
 )
 
+const BlockCacheSize = 1 << 30
+
 func OpenDB(home string, backendType dbm.BackendType) (dbm.DB, error) {
 	dataDir := filepath.Join(home, "data")
 	if backendType == dbm.RocksDBBackend {
-		// customize rocksdb options
-		db, err := grocksdb.OpenDb(NewRocksdbOptions(false), filepath.Join(dataDir, "application.db"))
-		if err != nil {
-			return nil, err
-		}
-		ro := grocksdb.NewDefaultReadOptions()
-		wo := grocksdb.NewDefaultWriteOptions()
-		woSync := grocksdb.NewDefaultWriteOptions()
-		woSync.SetSync(true)
-		return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+		return openRocksdb(filepath.Join(dataDir, "application.db"), false)
 	}
 
 	return dbm.NewDB("application", backendType, dataDir)
@@ -33,24 +27,58 @@ func OpenDB(home string, backendType dbm.BackendType) (dbm.DB, error) {
 func OpenReadOnlyDB(home string, backendType dbm.BackendType) (dbm.DB, error) {
 	dataDir := filepath.Join(home, "data")
 	if backendType == dbm.RocksDBBackend {
-		// customize rocksdb options
-		db, err := grocksdb.OpenDbForReadOnly(NewRocksdbOptions(false), filepath.Join(dataDir, "application.db"), false)
-		if err != nil {
-			return nil, err
-		}
-
-		ro := grocksdb.NewDefaultReadOptions()
-		wo := grocksdb.NewDefaultWriteOptions()
-		woSync := grocksdb.NewDefaultWriteOptions()
-		woSync.SetSync(true)
-		return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+		return openRocksdb(filepath.Join(dataDir, "application.db"), true)
 	}
 
 	return dbm.NewDB("application", backendType, dataDir)
 }
 
-func NewRocksdbOptions(sstFileWriter bool) *grocksdb.Options {
-	opts := grocksdb.NewDefaultOptions()
+func openRocksdb(dir string, readonly bool) (dbm.DB, error) {
+	opts, err := loadLatestOptions(dir)
+	if err != nil {
+		return nil, err
+	}
+	// customize rocksdb options
+	opts = NewRocksdbOptions(opts, false)
+
+	var db *grocksdb.DB
+	if readonly {
+		db, err = grocksdb.OpenDbForReadOnly(opts, dir, false)
+	} else {
+		db, err = grocksdb.OpenDb(opts, dir)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	ro := grocksdb.NewDefaultReadOptions()
+	wo := grocksdb.NewDefaultWriteOptions()
+	woSync := grocksdb.NewDefaultWriteOptions()
+	woSync.SetSync(true)
+	return dbm.NewRocksDBWithRawDB(db, ro, wo, woSync), nil
+}
+
+// loadLatestOptions try to load options from existing db, returns nil if not exists.
+func loadLatestOptions(dir string) (*grocksdb.Options, error) {
+	opts, err := grocksdb.LoadLatestOptions(dir, grocksdb.NewDefaultEnv(), true, grocksdb.NewLRUCache(BlockCacheSize))
+	if err != nil {
+		// not found is not an error
+		if strings.HasPrefix(err.Error(), "NotFound: ") {
+			return nil, nil
+		}
+		return nil, err
+	}
+	return opts.Options(), nil
+}
+
+// NewRocksdbOptions build options for `application.db`,
+// it overrides existing options if provided, otherwise create new one assuming it's a new database.
+func NewRocksdbOptions(opts *grocksdb.Options, sstFileWriter bool) *grocksdb.Options {
+	if opts == nil {
+		opts = grocksdb.NewDefaultOptions()
+		// only enable dynamic-level-bytes on new db, don't override for existing db
+		opts.SetLevelCompactionDynamicLevelBytes(true)
+	}
 	opts.SetCreateIfMissing(true)
 	opts.IncreaseParallelism(runtime.NumCPU())
 	opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024)
@@ -60,7 +88,7 @@ func NewRocksdbOptions(sstFileWriter bool) *grocksdb.Options {
 	bbto := grocksdb.NewDefaultBlockBasedTableOptions()
 
 	// 1G block cache
-	bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30))
+	bbto.SetBlockCache(grocksdb.NewLRUCache(BlockCacheSize))
 
 	// http://rocksdb.org/blog/2021/12/29/ribbon-filter.html
 	bbto.SetFilterPolicy(grocksdb.NewRibbonHybridFilterPolicy(9.9, 1))

diff --git a/go.mod b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/gorilla/mux v1.8.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
-	github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b
+	github.com/linxGnu/grocksdb v1.7.15
 	github.com/peggyjv/gravity-bridge/module/v2 v2.0.0-20220420162017-838c0d25e974
 	github.com/rakyll/statik v0.1.7
 	github.com/spf13/cast v1.5.0
@@ -205,7 +205,7 @@ replace (
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 
 	// use replace to force update grocksdb dependency in tm-db
-	github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b
+	github.com/linxGnu/grocksdb => github.com/linxGnu/grocksdb v1.7.16-0.20230303082518-587b6c4f0dab
 	github.com/miguelmota/go-ethereum-hdwallet => github.com/crypto-org-chain/go-ethereum-hdwallet v0.1.2
 
 	// TODO: remove when gravity update dependencies

diff --git a/go.sum b/go.sum
@@ -1631,8 +1631,8 @@ github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QT
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
-github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b h1:Olh1b4gdN5J9ZZyCiudSZ7HLrU+IL4SkYFcgyM9rGsA=
-github.com/linxGnu/grocksdb v1.7.15-0.20230222024938-b61261a9193b/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
+github.com/linxGnu/grocksdb v1.7.16-0.20230303082518-587b6c4f0dab h1:aTVNLqaxBD1ue7MqBEU5rkeVd8LyBJxzpf3EtkDEGiI=
+github.com/linxGnu/grocksdb v1.7.16-0.20230303082518-587b6c4f0dab/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4=
 github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=

diff --git a/gomod2nix.toml b/gomod2nix.toml
@@ -330,8 +330,8 @@ schema = 3
     version = "v0.1.0"
     hash = "sha256-wQqGTtRWsfR9n0O/SXHVgECebbnNmHddxJIbG63OJBQ="
   [mod."github.com/linxGnu/grocksdb"]
-    version = "v1.7.15-0.20230222024938-b61261a9193b"
-    hash = "sha256-Hry5mpO8WqCuYZ0zCnOt6kopDGprc7/nI318A2D+Kk0="
+    version = "v1.7.16-0.20230303082518-587b6c4f0dab"
+    hash = "sha256-MuH+rIx+r3zirXdKPHFRYeAP7QORr9gSwSiCJST7534="
     replaced = "github.com/linxGnu/grocksdb"
   [mod."github.com/magiconair/properties"]
     version = "v1.8.6"

diff --git a/versiondb/client/flags.go b/versiondb/client/flags.go
@@ -20,5 +20,4 @@ const (
 	flagLoadSnapshot     = "load-snapshot"
 	flagSorterChunkSize  = "sorter-chunk-size"
 	flagInitialVersion   = "initial-version"
-	flagNumLevels        = "num-levels"
 )
diff --git a/versiondb/client/ingest_sst.go b/versiondb/client/ingest_sst.go
@@ -20,18 +20,7 @@ func IngestVersionDBSSTCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			numLevels, err := cmd.Flags().GetInt(flagNumLevels)
-			if err != nil {
-				return err
-			}
 			opts := tsrocksdb.NewVersionDBOpts(false)
-			// it's a workaround for rocksdb issue(https://github.com/facebook/rocksdb/issues/11212),
-			// because rocksdb always ingest files into level `num_levels-1`,
-			// the new data will take a very long time to reach that level,
-			// level3 is the bottommost level in practice.
-			if numLevels > 0 {
-				opts.SetNumLevels(numLevels)
-			}
 			db, cfHandle, err := tsrocksdb.OpenVersionDBWithOpts(dbPath, opts)
 			if err != nil {
 				return err
@@ -67,6 +56,5 @@ func IngestVersionDBSSTCmd() *cobra.Command {
 	}
 	cmd.Flags().Bool(flagMoveFiles, false, "move sst files instead of copy them")
 	cmd.Flags().Int64(flagMaximumVersion, 0, "Specify the maximum version covered by the ingested files, if it's bigger than existing recorded latest version, will update it.")
-	cmd.Flags().Int(flagNumLevels, 4, "Override the num levels in default options, when ingesting into a new db, the sst files will be ingested into level `num-levels-1`, set to 0 to not override.")
 	return cmd
 }
diff --git a/versiondb/client/restore_app_db.go b/versiondb/client/restore_app_db.go
@@ -63,10 +63,6 @@ func RestoreAppDBCmd(opts Options) *cobra.Command {
 			if err != nil {
 				return err
 			}
-			numLevels, err := cmd.Flags().GetInt(flagNumLevels)
-			if err != nil {
-				return err
-			}
 			stores, err := GetStoresOrDefault(cmd, opts.DefaultStores)
 			if err != nil {
 				return err
@@ -148,13 +144,6 @@ func RestoreAppDBCmd(opts Options) *cobra.Command {
 			ingestOpts.SetMoveFiles(true)
 
 			opts := opts.AppRocksDBOptions(false)
-			// it's a workaround for rocksdb issue(https://github.com/facebook/rocksdb/issues/11212),
-			// because rocksdb always ingest files into level `num_levels-1`,
-			// the new data will take a very long time to reach that level,
-			// level3 is the bottommost level in practice.
-			if numLevels > 0 {
-				opts.SetNumLevels(numLevels)
-			}
 			db, err := grocksdb.OpenDb(opts, iavlDir)
 			if err != nil {
 				return errors.Wrap(err, "open iavl db fail")
@@ -179,7 +168,6 @@ func RestoreAppDBCmd(opts Options) *cobra.Command {
 	cmd.Flags().String(flagStores, "", "list of store names, default to the current store list in application")
 	cmd.Flags().Uint64(flagSorterChunkSize, DefaultSorterChunkSizeIAVL, "uncompressed chunk size for external sorter, it decides the peak ram usage, on disk it'll be snappy compressed")
 	cmd.Flags().Int(flagConcurrency, runtime.NumCPU(), "Number concurrent goroutines to parallelize the work")
-	cmd.Flags().Int(flagNumLevels, 4, "Override the num levels in default options, when ingesting into a new db, the sst files will be ingested into level `num-levels-1`, set to 0 to not override.")
 
 	return cmd
 }

diff --git a/versiondb/go.mod b/versiondb/go.mod
@@ -11,7 +11,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/ledgerwatch/erigon-lib v0.0.0-20230130172141-b594e6cdbf76
-	github.com/linxGnu/grocksdb v1.7.14
+	github.com/linxGnu/grocksdb v1.7.15
 	github.com/spf13/cast v1.5.0
 	github.com/spf13/cobra v1.6.1
 	github.com/stretchr/testify v1.8.1

diff --git a/versiondb/go.sum b/versiondb/go.sum
@@ -604,8 +604,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6
 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
-github.com/linxGnu/grocksdb v1.7.14 h1:8lMZzyWeNP5lI0BIppX05DzmQzXj/Tgu82bgWYtowLY=
-github.com/linxGnu/grocksdb v1.7.14/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
+github.com/linxGnu/grocksdb v1.7.15 h1:AEhP28lkeAybv5UYNYviYISpR6bJejEnKuYbnWAnxx0=
+github.com/linxGnu/grocksdb v1.7.15/go.mod h1:pY55D0o+r8yUYLq70QmhdudxYvoDb9F+9puf4m3/W+U=
 github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=