Make rocksdb configurable (backport #1658)

.github/workflows/ci-rocksdb-build.yml

@@ -1,5 +1,8 @@
 name: Continuous Integration (Rocksdb Build)
 
+env:
+  ROCKSDB_VERSION: v8.1.1
+
 on:
   workflow_call:
 jobs:
@@ -16,7 +19,29 @@ jobs:
           cache: true
       - name: build rocksdb dependency
         run: bash ${GITHUB_WORKSPACE}/.github/scripts/install-rocksdb.sh
-        env:
-          ROCKSDB_VERSION: v7.10.2
       - name: build application
         run: make build COSMOS_BUILD_OPTIONS=rocksdb
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: install RocksDB dependencies
+        run: sudo apt-get update
+          && sudo apt-get install -y git make gcc libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+      - name: install RocksDB as shared library
+        run: git clone https://github.com/facebook/rocksdb.git
+          && cd rocksdb
+          && git checkout $ROCKSDB_VERSION
+          && sudo make -j$(nproc) install-shared
+          && sudo ldconfig
+      - name: checkout repo from current commit
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.20"
+          check-latest: true
+          cache: true
+      - name: run unit tests
+        run: make test-rocksdb

Dockerfile-rocksdb

@@ -10,7 +10,7 @@ WORKDIR /root
 # default home directory is /root
 
 # install rocksdb
-ARG rocksdb_version=v7.10.2
+ARG rocksdb_version=v8.1.1
 ENV ROCKSDB_VERSION=$rocksdb_version
 
 RUN git clone https://github.com/facebook/rocksdb.git \

Makefile

@@ -302,6 +302,9 @@ test-e2e: docker-build
 test:
 	@go test $$(go list ./... | grep -v 'contrib' | grep -v 'tests/e2e')
 
+test-rocksdb:
+	@go test -tags=rocksdb ./cmd/kava/opendb
+
 # Run cli integration tests
 # `-p 4` to use 4 cores, `-tags cli_test` to tell go not to ignore the cli package
 # These tests use the `kvd` or `kvcli` binaries in the build dir, or in `$BUILDDIR` if that env var is set.

cmd/kava/cmd/root.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kava-labs/kava/app"
 	"github.com/kava-labs/kava/app/params"
 	kavaclient "github.com/kava-labs/kava/client"
+	"github.com/kava-labs/kava/cmd/kava/opendb"
 )
 
 // EnvPrefix is the prefix environment variables must have to configure the app.
@@ -105,13 +106,15 @@ func addSubCmds(rootCmd *cobra.Command, encodingConfig params.EncodingConfig, de
 		encodingConfig: encodingConfig,
 	}
 
+	opts := ethermintserver.StartOptions{
+		AppCreator:      ac.newApp,
+		DefaultNodeHome: app.DefaultNodeHome,
+		DBOpener:        opendb.OpenDB,
+	}
 	// ethermintserver adds additional flags to start the JSON-RPC server for evm support
 	ethermintserver.AddCommands(
 		rootCmd,
-		ethermintserver.NewDefaultStartOptions(
-			ac.newApp,
-			app.DefaultNodeHome,
-		),
+		opts,
 		ac.appExport,
 		ac.addStartCmdFlags,
 	)

cmd/kava/opendb/metrics.go

@@ -0,0 +1,166 @@
+//go:build rocksdb
+// +build rocksdb
+
+package opendb
+
+import (
+	"github.com/go-kit/kit/metrics"
+	"github.com/go-kit/kit/metrics/prometheus"
+	stdprometheus "github.com/prometheus/client_golang/prometheus"
+)
+
+// rocksdbMetrics will be initialized in registerMetrics() if enableRocksdbMetrics flag set to true
+var rocksdbMetrics *Metrics
+
+// Metrics contains all rocksdb metrics which will be reported to prometheus
+type Metrics struct {
+	// Keys
+	NumberKeysWritten metrics.Gauge
+	NumberKeysRead    metrics.Gauge
+	NumberKeysUpdated metrics.Gauge
+	EstimateNumKeys   metrics.Gauge
+
+	// Files
+	NumberFileOpens  metrics.Gauge
+	NumberFileErrors metrics.Gauge
+
+	// Memory
+	BlockCacheUsage         metrics.Gauge
+	EstimateTableReadersMem metrics.Gauge
+	CurSizeAllMemTables     metrics.Gauge
+	BlockCachePinnedUsage   metrics.Gauge
+
+	// Cache
+	BlockCacheMiss        metrics.Gauge
+	BlockCacheHit         metrics.Gauge
+	BlockCacheAdd         metrics.Gauge
+	BlockCacheAddFailures metrics.Gauge
+}
+
+// registerMetrics registers metrics in prometheus and initializes rocksdbMetrics variable
+func registerMetrics() {
+	if rocksdbMetrics != nil {
+		// metrics already registered
+		return
+	}
+
+	labels := make([]string, 0)
+	rocksdbMetrics = &Metrics{
+		// Keys
+		NumberKeysWritten: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_written",
+			Help:      "",
+		}, labels),
+		NumberKeysRead: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_read",
+			Help:      "",
+		}, labels),
+		NumberKeysUpdated: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "number_keys_updated",
+			Help:      "",
+		}, labels),
+		EstimateNumKeys: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "key",
+			Name:      "estimate_num_keys",
+			Help:      "estimated number of total keys in the active and unflushed immutable memtables and storage",
+		}, labels),
+
+		// Files
+		NumberFileOpens: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "file",
+			Name:      "number_file_opens",
+			Help:      "",
+		}, labels),
+		NumberFileErrors: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "file",
+			Name:      "number_file_errors",
+			Help:      "",
+		}, labels),
+
+		// Memory
+		BlockCacheUsage: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "block_cache_usage",
+			Help:      "memory size for the entries residing in block cache",
+		}, labels),
+		EstimateTableReadersMem: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "estimate_table_readers_mem",
+			Help:      "estimated memory used for reading SST tables, excluding memory used in block cache (e.g., filter and index blocks)",
+		}, labels),
+		CurSizeAllMemTables: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "cur_size_all_mem_tables",
+			Help:      "approximate size of active and unflushed immutable memtables (bytes)",
+		}, labels),
+		BlockCachePinnedUsage: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "memory",
+			Name:      "block_cache_pinned_usage",
+			Help:      "returns the memory size for the entries being pinned",
+		}, labels),
+
+		// Cache
+		BlockCacheMiss: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_miss",
+			Help:      "block_cache_miss == block_cache_index_miss + block_cache_filter_miss + block_cache_data_miss",
+		}, labels),
+		BlockCacheHit: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_hit",
+			Help:      "block_cache_hit == block_cache_index_hit + block_cache_filter_hit + block_cache_data_hit",
+		}, labels),
+		BlockCacheAdd: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_add",
+			Help:      "number of blocks added to block cache",
+		}, labels),
+		BlockCacheAddFailures: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{
+			Namespace: "rocksdb",
+			Subsystem: "cache",
+			Name:      "block_cache_add_failures",
+			Help:      "number of failures when adding blocks to block cache",
+		}, labels),
+	}
+}
+
+// report reports metrics to prometheus based on rocksdb props and stats
+func (m *Metrics) report(props *properties, stats *stats) {
+	// Keys
+	m.NumberKeysWritten.Set(float64(stats.NumberKeysWritten))
+	m.NumberKeysRead.Set(float64(stats.NumberKeysRead))
+	m.NumberKeysUpdated.Set(float64(stats.NumberKeysUpdated))
+	m.EstimateNumKeys.Set(float64(props.EstimateNumKeys))
+
+	// Files
+	m.NumberFileOpens.Set(float64(stats.NumberFileOpens))
+	m.NumberFileErrors.Set(float64(stats.NumberFileErrors))
+
+	// Memory
+	m.BlockCacheUsage.Set(float64(props.BlockCacheUsage))
+	m.EstimateTableReadersMem.Set(float64(props.EstimateTableReadersMem))
+	m.CurSizeAllMemTables.Set(float64(props.CurSizeAllMemTables))
+	m.BlockCachePinnedUsage.Set(float64(props.BlockCachePinnedUsage))
+
+	// Cache
+	m.BlockCacheMiss.Set(float64(stats.BlockCacheMiss))
+	m.BlockCacheHit.Set(float64(stats.BlockCacheHit))
+	m.BlockCacheAdd.Set(float64(stats.BlockCacheAdd))
+	m.BlockCacheAddFailures.Set(float64(stats.BlockCacheAddFailures))
+}

cmd/kava/opendb/opendb.go

@@ -0,0 +1,18 @@
+//go:build !rocksdb
+// +build !rocksdb
+
+package opendb
+
+import (
+	"path/filepath"
+
+	"github.com/cosmos/cosmos-sdk/server/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+// OpenDB is a copy of default DBOpener function used by ethermint, see for details:
+// https://github.com/evmos/ethermint/blob/07cf2bd2b1ce9bdb2e44ec42a39e7239292a14af/server/start.go#L647
+func OpenDB(_ types.AppOptions, home string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(home, "data")
+	return dbm.NewDB("application", backendType, dataDir)
+}