feat: rocksdb support auto tune with default config

apache · May 6, 2023 · 7348179 · 7348179
1 parent f416fbb
commit 7348179
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 1 deletion.
diff --git a/kvrocks.conf b/kvrocks.conf
@@ -833,5 +833,13 @@ rocksdb.write_options.low_pri no
 # Default: no
 rocksdb.write_options.memtable_insert_hint_per_batch no
 
+
+# Support RocksDB auto-tune rate limiter for the background IO
+# if enabled, Rate limiter will limit the compaction write if flush write is high
+# Please see https://rocksdb.org/blog/2017/12/18/17-auto-tuned-rate-limiter.html
+#
+# Default: yes
+rocksdb.rate_limiter_auto_tuned yes
+
 ################################ NAMESPACE #####################################
 # namespace.test change.me
diff --git a/src/config/config.cc b/src/config/config.cc
@@ -210,6 +210,7 @@ Config::Config() {
       {"rocksdb.level_compaction_dynamic_level_bytes", false,
        new YesNoField(&rocks_db.level_compaction_dynamic_level_bytes, false)},
       {"rocksdb.max_background_jobs", false, new IntField(&rocks_db.max_background_jobs, 4, 0, 32)},
+      {"rocksdb.rate_limiter_auto_tuned", true, new YesNoField(&rocks_db.rate_limiter_auto_tuned, true)},
 
       /* rocksdb write options */
       {"rocksdb.write_options.sync", true, new YesNoField(&rocks_db.write_options.sync, false)},

diff --git a/src/config/config.h b/src/config/config.h
@@ -187,6 +187,7 @@ struct Config {
     int max_bytes_for_level_multiplier;
     bool level_compaction_dynamic_level_bytes;
     int max_background_jobs;
+    bool rate_limiter_auto_tuned;
 
     struct WriteOptions {
       bool sync;

diff --git a/src/storage/storage.cc b/src/storage/storage.cc
@@ -159,8 +159,19 @@ rocksdb::Options Storage::InitRocksDBOptions() {
   options.sst_file_manager = sst_file_manager_;
   int64_t max_io_mb = kIORateLimitMaxMb;
   if (config_->max_io_mb > 0) max_io_mb = config_->max_io_mb;
-  rate_limiter_ =
+  if (config_->rocks_db.rate_limiter_auto_tuned) {
+    rate_limiter_ =
+      std::shared_ptr<rocksdb::RateLimiter>(rocksdb::NewGenericRateLimiter(
+            max_io_mb * static_cast<int64_t>(MiB),
+            100 * 1000, /* default */
+            10, /* default */
+            rocksdb::RateLimiter::Mode::kWritesOnly,
+            true
+          ));
+  } else {
+    rate_limiter_ =
       std::shared_ptr<rocksdb::RateLimiter>(rocksdb::NewGenericRateLimiter(max_io_mb * static_cast<int64_t>(MiB)));
+  }
   options.rate_limiter = rate_limiter_;
   options.delayed_write_rate = static_cast<uint64_t>(config_->rocks_db.delayed_write_rate);
   options.compaction_readahead_size = static_cast<size_t>(config_->rocks_db.compaction_readahead_size);

diff --git a/tests/cppunit/config_test.cc b/tests/cppunit/config_test.cc
@@ -79,6 +79,7 @@ TEST(Config, GetAndSet) {
       {"rocksdb.max_bytes_for_level_multiplier", "10"},
       {"rocksdb.level_compaction_dynamic_level_bytes", "yes"},
       {"rocksdb.max_background_jobs", "4"},
+      {"rocksdb.rate_limiter_auto_tuned", "yes"},
   };
   std::vector<std::string> values;
   for (const auto &iter : mutable_cases) {