(feat) Supports partitioned index filters settings for BlockBasedOpti…

…ons (rust-rocksdb#294)

This PR exports more options in RocksDB `BlockBasedTableOptions`, because we want to use [Partitioned Index Filters](https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters#how-to-use-it).

There is an issue [cache_index_and_filter_blocks causes Get slowdown](facebook/rocksdb#3961) in rocksdb, and we test the recommend `Partitioned Index Filters` in java JNI binding, it works well.

So i think these options should be exported to rust-rocksdb users too.

The rocksdb log in test case `test_partitioned_index_filters` as below:

```
  cache_index_and_filter_blocks: 1
  cache_index_and_filter_blocks_with_high_priority: 1
  pin_l0_filter_and_index_blocks_in_cache: 1
  pin_top_level_index_and_filter: 1
  index_type: 2
  hash_index_allow_collision: 1
  checksum: 1
  no_block_cache: 0
  block_cache: 0x7ff4540060d0
  block_cache_name: LRUCache
  block_cache_options:
    capacity : 8388608
    num_shard_bits : 4
    strict_capacity_limit : 0
    memory_allocator : None
    high_pri_pool_ratio: 0.000
  block_cache_compressed: (nil)
  persistent_cache: (nil)
  block_size: 4096
  block_size_deviation: 10
  block_restart_interval: 16
  index_block_restart_interval: 1
  metadata_block_size: 4096
  partition_filters: 1
  use_delta_encoding: 1
  filter_policy: rocksdb.BuiltinBloomFilter
  whole_key_filtering: 1
  verify_compression: 0
  read_amp_bytes_per_bit: 0
  format_version: 2
  enable_index_compression: 1
  block_align: 0
```

It looks working.

Thanks you for this great project.

librocksdb_sys/crocksdb/c.cc

@@ -1689,6 +1689,11 @@ void crocksdb_block_based_options_destroy(
   delete options;
 }
 
+void crocksdb_block_based_options_set_metadata_block_size(
+    crocksdb_block_based_table_options_t* options, size_t block_size) {
+  options->rep.metadata_block_size = block_size;
+}
+
 void crocksdb_block_based_options_set_block_size(
     crocksdb_block_based_table_options_t* options, size_t block_size) {
   options->rep.block_size = block_size;
@@ -1752,11 +1757,21 @@ void crocksdb_block_based_options_set_hash_index_allow_collision(
   options->rep.hash_index_allow_collision = v;
 }
 
+void crocksdb_block_based_options_set_partition_filters(
+    crocksdb_block_based_table_options_t* options, unsigned char v) {
+  options->rep.partition_filters = v;
+}
+
 void crocksdb_block_based_options_set_cache_index_and_filter_blocks(
     crocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.cache_index_and_filter_blocks = v;
 }
 
+void crocksdb_block_based_options_set_pin_top_level_index_and_filter(
+    crocksdb_block_based_table_options_t* options, unsigned char v) {
+  options->rep.pin_top_level_index_and_filter = v;
+}
+
 void crocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(
     crocksdb_block_based_table_options_t* options, unsigned char v) {
   options->rep.cache_index_and_filter_blocks_with_high_priority = v;

librocksdb_sys/crocksdb/crocksdb/c.h

@@ -610,6 +610,8 @@ extern C_ROCKSDB_LIBRARY_API crocksdb_block_based_table_options_t*
 crocksdb_block_based_options_create();
 extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_destroy(
     crocksdb_block_based_table_options_t* options);
+extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_set_metadata_block_size(
+    crocksdb_block_based_table_options_t* options, size_t block_size);
 extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_set_block_size(
     crocksdb_block_based_table_options_t* options, size_t block_size);
 extern C_ROCKSDB_LIBRARY_API void
@@ -637,16 +639,23 @@ extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_set_format_versio
 enum {
   crocksdb_block_based_table_index_type_binary_search = 0,
   crocksdb_block_based_table_index_type_hash_search = 1,
+  crocksdb_block_based_table_index_type_two_level_index_search = 2,
 };
 extern C_ROCKSDB_LIBRARY_API void crocksdb_block_based_options_set_index_type(
     crocksdb_block_based_table_options_t*, int);  // uses one of the above enums
 extern C_ROCKSDB_LIBRARY_API void
 crocksdb_block_based_options_set_hash_index_allow_collision(
     crocksdb_block_based_table_options_t*, unsigned char);
 extern C_ROCKSDB_LIBRARY_API void
+crocksdb_block_based_options_set_partition_filters(
+    crocksdb_block_based_table_options_t*, unsigned char);
+extern C_ROCKSDB_LIBRARY_API void
 crocksdb_block_based_options_set_cache_index_and_filter_blocks(
     crocksdb_block_based_table_options_t*, unsigned char);
 extern C_ROCKSDB_LIBRARY_API void
+crocksdb_block_based_options_set_pin_top_level_index_and_filter(
+    crocksdb_block_based_table_options_t*, unsigned char);
+extern C_ROCKSDB_LIBRARY_API void
 crocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(
     crocksdb_block_based_table_options_t*, unsigned char);
 extern C_ROCKSDB_LIBRARY_API void

librocksdb_sys/src/lib.rs

@@ -281,6 +281,14 @@ pub enum DBTitanDBBlobRunMode {
     Fallback = 2,
 }
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+#[repr(C)]
+pub enum IndexType {
+    BinarySearch = 0,
+    HashSearch = 1,
+    TwoLevelIndexSearch = 2,
+}
+
 pub fn error_message(ptr: *mut c_char) -> String {
     let c_str = unsafe { CStr::from_ptr(ptr) };
     let s = format!("{}", c_str.to_string_lossy());
@@ -356,6 +364,10 @@ extern "C" {
 
     pub fn crocksdb_block_based_options_create() -> *mut DBBlockBasedTableOptions;
     pub fn crocksdb_block_based_options_destroy(opts: *mut DBBlockBasedTableOptions);
+    pub fn crocksdb_block_based_options_set_metadata_block_size(
+        block_options: *mut DBBlockBasedTableOptions,
+        block_size: size_t,
+    );
     pub fn crocksdb_block_based_options_set_block_size(
         block_options: *mut DBBlockBasedTableOptions,
         block_size: size_t,
@@ -368,10 +380,26 @@ extern "C" {
         block_options: *mut DBBlockBasedTableOptions,
         block_restart_interval: c_int,
     );
+    pub fn crocksdb_block_based_options_set_index_type(
+        block_options: *mut DBBlockBasedTableOptions,
+        v: IndexType,
+    );
+    pub fn crocksdb_block_based_options_set_hash_index_allow_collision(
+        block_options: *mut DBBlockBasedTableOptions,
+        v: c_uchar,
+    );
+    pub fn crocksdb_block_based_options_set_partition_filters(
+        block_options: *mut DBBlockBasedTableOptions,
+        v: c_uchar,
+    );
     pub fn crocksdb_block_based_options_set_cache_index_and_filter_blocks(
         block_options: *mut DBBlockBasedTableOptions,
         v: c_uchar,
     );
+    pub fn crocksdb_block_based_options_set_pin_top_level_index_and_filter(
+        block_options: *mut DBBlockBasedTableOptions,
+        v: c_uchar,
+    );
     pub fn crocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(
         block_options: *mut DBBlockBasedTableOptions,
         v: c_uchar,

src/lib.rs

@@ -28,7 +28,7 @@ pub use librocksdb_sys::{
     self as crocksdb_ffi, new_bloom_filter, CompactionPriority, CompactionReason,
     DBBottommostLevelCompaction, DBCompactionStyle, DBCompressionType, DBEntryType, DBInfoLogLevel,
     DBRateLimiterMode, DBRecoveryMode, DBStatisticsHistogramType, DBStatisticsTickerType,
-    DBTitanDBBlobRunMode, WriteStallCondition,
+    DBTitanDBBlobRunMode, IndexType, WriteStallCondition,
 };
 pub use merge_operator::MergeOperands;
 pub use metadata::{ColumnFamilyMetaData, LevelMetaData, SstFileMetaData};

src/rocksdb_options.rs

@@ -20,7 +20,8 @@ use crocksdb_ffi::{
     DBCompactionOptions, DBCompressionType, DBFifoCompactionOptions, DBFlushOptions,
     DBInfoLogLevel, DBInstance, DBLRUCacheOptions, DBRateLimiter, DBRateLimiterMode, DBReadOptions,
     DBRecoveryMode, DBRestoreOptions, DBSnapshot, DBStatisticsHistogramType,
-    DBStatisticsTickerType, DBTitanDBOptions, DBTitanReadOptions, DBWriteOptions, Options,
+    DBStatisticsTickerType, DBTitanDBOptions, DBTitanReadOptions, DBWriteOptions, IndexType,
+    Options,
 };
 use event_listener::{new_event_listener, EventListener};
 use libc::{self, c_double, c_int, c_uchar, c_void, size_t};
@@ -80,12 +81,24 @@ impl BlockBasedOptions {
         BlockBasedOptions::default()
     }
 
+    pub fn set_metadata_block_size(&mut self, size: usize) {
+        unsafe {
+            crocksdb_ffi::crocksdb_block_based_options_set_metadata_block_size(self.inner, size);
+        }
+    }
+
     pub fn set_block_size(&mut self, size: usize) {
         unsafe {
             crocksdb_ffi::crocksdb_block_based_options_set_block_size(self.inner, size);
         }
     }
 
+    pub fn set_index_type(&mut self, index_type: IndexType) {
+        unsafe {
+            crocksdb_ffi::crocksdb_block_based_options_set_index_type(self.inner, index_type);
+        }
+    }
+
     pub fn set_block_cache(&mut self, cache: &Cache) {
         unsafe {
             crocksdb_ffi::crocksdb_block_based_options_set_block_cache(self.inner, cache.inner);
@@ -110,6 +123,20 @@ impl BlockBasedOptions {
         }
     }
 
+    pub fn set_hash_index_allow_collision(&mut self, v: bool) {
+        unsafe {
+            crocksdb_ffi::crocksdb_block_based_options_set_hash_index_allow_collision(
+                self.inner, v as u8,
+            );
+        }
+    }
+
+    pub fn set_partition_filters(&mut self, v: bool) {
+        unsafe {
+            crocksdb_ffi::crocksdb_block_based_options_set_partition_filters(self.inner, v as u8);
+        }
+    }
+
     pub fn set_cache_index_and_filter_blocks(&mut self, v: bool) {
         unsafe {
             crocksdb_ffi::crocksdb_block_based_options_set_cache_index_and_filter_blocks(
@@ -118,6 +145,14 @@ impl BlockBasedOptions {
         }
     }
 
+    pub fn set_pin_top_level_index_and_filter(&mut self, v: bool) {
+        unsafe {
+            crocksdb_ffi::crocksdb_block_based_options_set_pin_top_level_index_and_filter(
+                self.inner, v as u8,
+            );
+        }
+    }
+
     pub fn set_cache_index_and_filter_blocks_with_high_priority(&mut self, v: bool) {
         unsafe {
             crocksdb_ffi::

tests/cases/test_rocksdb_options.rs

@@ -17,8 +17,8 @@ use rocksdb::crocksdb_ffi::{
 };
 use rocksdb::{
     BlockBasedOptions, Cache, ColumnFamilyOptions, CompactOptions, DBOptions, Env,
-    FifoCompactionOptions, LRUCacheOptions, ReadOptions, SeekKey, SliceTransform, Writable,
-    WriteOptions, DB,
+    FifoCompactionOptions, IndexType, LRUCacheOptions, ReadOptions, SeekKey, SliceTransform,
+    Writable, WriteOptions, DB,
 };
 use std::path::Path;
 use std::sync::Arc;
@@ -279,6 +279,31 @@ fn test_set_pin_l0_filter_and_index_blocks_in_cache() {
     .unwrap();
 }
 
+#[test]
+fn test_partitioned_index_filters() {
+    let path = TempDir::new("_rust_rocksdb_set_cache_and_index").expect("");
+    let mut opts = DBOptions::new();
+    let mut cf_opts = ColumnFamilyOptions::new();
+    opts.create_if_missing(true);
+    let mut block_opts = BlockBasedOptions::new();
+    // See https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters#how-to-use-it
+    block_opts.set_index_type(IndexType::TwoLevelIndexSearch);
+    block_opts.set_partition_filters(true);
+    block_opts.set_bloom_filter(10, false);
+    block_opts.set_metadata_block_size(4096);
+    block_opts.set_cache_index_and_filter_blocks(true);
+    block_opts.set_pin_top_level_index_and_filter(true);
+    block_opts.set_cache_index_and_filter_blocks_with_high_priority(true);
+    block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);
+    cf_opts.set_block_based_table_factory(&block_opts);
+    DB::open_cf(
+        opts,
+        path.path().to_str().unwrap(),
+        vec![("default", cf_opts)],
+    )
+    .unwrap();
+}
+
 #[test]
 fn test_set_lru_cache() {
     let path = TempDir::new("_rust_rocksdb_set_set_lru_cache").expect("");