Metadata and subkey column family share a singleblock cache (apache#365)

kvrocks.conf

@@ -328,6 +328,13 @@ rocksdb.metadata_block_cache_size 2048
 # Default: 2048MB
 rocksdb.subkey_block_cache_size 2048
 
+# Metadata column family and subkey column family will share a single block cache
+# if set 'yes'. The capacity of shared block cache is
+# metadata_block_cache_size + subkey_block_cache_size
+#
+# Default: no
+rocksdb.share_metadata_and_subkey_block_cache no
+
 # Number of open files that can be used by the DB.  You may need to
 # increase this if your database has a large working set. Value -1 means
 # files opened are always kept open. You can estimate number of files based

src/config.cc

@@ -122,6 +122,8 @@ Config::Config() {
       {"rocksdb.cache_index_and_filter_blocks", true, new YesNoField(&RocksDB.cache_index_and_filter_blocks, false)},
       {"rocksdb.subkey_block_cache_size", true, new IntField(&RocksDB.subkey_block_cache_size, 2048, 0, INT_MAX)},
       {"rocksdb.metadata_block_cache_size", true, new IntField(&RocksDB.metadata_block_cache_size, 2048, 0, INT_MAX)},
+      {"rocksdb.share_metadata_and_subkey_block_cache",
+       true, new YesNoField(&RocksDB.share_metadata_and_subkey_block_cache, false)},
       {"rocksdb.compaction_readahead_size", false, new IntField(&RocksDB.compaction_readahead_size, 2*MiB, 0, 64*MiB)},
       {"rocksdb.level0_slowdown_writes_trigger",
        false, new IntField(&RocksDB.level0_slowdown_writes_trigger, 20, 1, 1024)},

src/config.h

@@ -100,6 +100,7 @@ struct Config{
     bool cache_index_and_filter_blocks;
     int metadata_block_cache_size;
     int subkey_block_cache_size;
+    bool share_metadata_and_subkey_block_cache;
     int max_open_files;
     int write_buffer_size;
     int max_write_buffer_number;

src/storage.cc

@@ -153,15 +153,25 @@ Status Storage::Open(bool read_only) {
   size_t block_size = static_cast<size_t>(config_->RocksDB.block_size);
   size_t metadata_block_cache_size = config_->RocksDB.metadata_block_cache_size*MiB;
   size_t subkey_block_cache_size = config_->RocksDB.subkey_block_cache_size*MiB;
+
   rocksdb::Options options;
   InitOptions(&options);
   CreateColumnFamilies(options);
+
+  std::shared_ptr<rocksdb::Cache> shared_block_cache;
+  if (config_->RocksDB.share_metadata_and_subkey_block_cache) {
+    size_t shared_block_cache_size = metadata_block_cache_size + subkey_block_cache_size;
+    shared_block_cache = rocksdb::NewLRUCache(shared_block_cache_size, -1, false, 0.75);
+  }
+
   rocksdb::BlockBasedTableOptions metadata_table_opts;
   metadata_table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
-  metadata_table_opts.block_cache = rocksdb::NewLRUCache(metadata_block_cache_size, -1, false, 0.75);
+  metadata_table_opts.block_cache = shared_block_cache ?
+    shared_block_cache : rocksdb::NewLRUCache(metadata_block_cache_size, -1, false, 0.75);
   metadata_table_opts.cache_index_and_filter_blocks = cache_index_and_filter_blocks;
   metadata_table_opts.cache_index_and_filter_blocks_with_high_priority = true;
   metadata_table_opts.block_size = block_size;
+
   rocksdb::ColumnFamilyOptions metadata_opts(options);
   metadata_opts.table_factory.reset(rocksdb::NewBlockBasedTableFactory(metadata_table_opts));
   metadata_opts.compaction_filter_factory = std::make_shared<MetadataFilterFactory>(this);
@@ -171,7 +181,8 @@ Status Storage::Open(bool read_only) {
 
   rocksdb::BlockBasedTableOptions subkey_table_opts;
   subkey_table_opts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, true));
-  subkey_table_opts.block_cache = rocksdb::NewLRUCache(subkey_block_cache_size, -1, false, 0.75);
+  subkey_table_opts.block_cache = shared_block_cache ?
+    shared_block_cache : rocksdb::NewLRUCache(subkey_block_cache_size, -1, false, 0.75);
   subkey_table_opts.cache_index_and_filter_blocks = cache_index_and_filter_blocks;
   subkey_table_opts.cache_index_and_filter_blocks_with_high_priority = true;
   subkey_table_opts.block_size = block_size;