block base dopts

README.md

@@ -94,9 +94,6 @@ $ cargo test -- --test-threads=1
 
 
 ### TODO
- - [X] actix-web 3.0 - fixed mem leak
- - [X] channel for expire
- - [ ] support for more rocksDb options in config (bloom, block cache..)
  - [ ] fix running test in parallel (TODO per test db root path)
  - [ ] db iterator for on startup init
  - [ ] test compaction

config/db_config.toml

@@ -13,4 +13,11 @@ level_zero_stop_writes_trigger = 24
 level_zero_slowdown_writes_trigger = 20
 compaction_style = "Level"
 #block based
+use_cache = true
+cache_defaults = true
+cache_size = 100000
+block_cache = 10000
+bloom_filter_bits_per_key = 8
+bloom_filter_block_based = true
+index_type = "HashSearch"
 

src/config.rs

@@ -1,5 +1,5 @@
 use confy::ConfyError;
-use rocksdb::{DBCompactionStyle, Options};
+use rocksdb::{BlockBasedIndexType, BlockBasedOptions, Cache, DBCompactionStyle, Options};
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -68,6 +68,13 @@ pub struct RocksDbConfig {
     level_zero_stop_writes_trigger: i32,
     level_zero_slowdown_writes_trigger: i32,
     compaction_style: String,
+    use_cache: bool,
+    cache_defaults: bool,
+    cache_size: usize,
+    block_cache: usize,
+    bloom_filter_bits_per_key: i32,
+    bloom_filter_block_based: bool,
+    index_type: String,
 }
 
 impl Default for RocksDbConfig {
@@ -87,6 +94,13 @@ impl Default for RocksDbConfig {
             level_zero_stop_writes_trigger: 24,
             level_zero_slowdown_writes_trigger: 24,
             compaction_style: "Level".to_string(),
+            use_cache: true,
+            cache_defaults: true,
+            cache_size: 100_000,
+            block_cache: 10_000,
+            bloom_filter_bits_per_key: 8,
+            bloom_filter_block_based: true,
+            index_type: "HashSearch".to_string(),
         }
     }
 }
@@ -117,11 +131,30 @@ impl RocksDbConfig {
         opts.set_level_zero_slowdown_writes_trigger(self.level_zero_slowdown_writes_trigger);
         opts.set_compaction_style(get_compaction_style(&self.compaction_style));
         opts.create_if_missing(true);
+        opts.set_block_based_table_factory(&block_based_opts(self));
 
         opts
     }
 }
 
+fn block_based_opts(cfg: &RocksDbConfig) -> BlockBasedOptions {
+    let mut opts = BlockBasedOptions::default();
+    if cfg.cache_defaults {
+        opts
+    } else if !cfg.use_cache {
+        opts.disable_cache();
+        opts
+    } else {
+        opts.set_block_cache(
+            &Cache::new_lru_cache(cfg.cache_size).expect("Could not create Cache"),
+        );
+        opts.set_block_size(cfg.block_cache);
+        opts.set_bloom_filter(cfg.bloom_filter_bits_per_key, cfg.bloom_filter_block_based);
+        opts.set_index_type(get_index_type(&cfg.index_type));
+        opts
+    }
+}
+
 pub fn load_db_config(cfg_path: &str) -> Result<DbConfig, ConfyError> {
     let rocks_cfg = confy::load_path(format!("{}/db_config.toml", cfg_path))?;
     Ok(DbConfig::new(rocks_cfg))
@@ -149,3 +182,18 @@ fn get_compaction_style(s: &str) -> DBCompactionStyle {
         }
     }
 }
+
+fn get_index_type(s: &str) -> BlockBasedIndexType {
+    match s.to_lowercase().as_str() {
+        "hashsearch" => BlockBasedIndexType::HashSearch,
+        "binarysearch" => BlockBasedIndexType::BinarySearch,
+        "twolevelindexsearch" => BlockBasedIndexType::TwoLevelIndexSearch,
+        _ => {
+            error!(
+                "Unknown block based index type {} - fallback to default {}",
+                &s, "HashSearch"
+            );
+            BlockBasedIndexType::HashSearch
+        }
+    }
+}

src/db.rs

@@ -149,7 +149,6 @@ impl DbManager {
             .name("async-expire-thread".into())
             .spawn(move || {
                 for boxed in rx {
-                    info!("expiring key in thead {:?}", thread::current());
                     boxed.invoke()
                 }
             })

src/integration_tests.rs

@@ -9,7 +9,6 @@ use crate::config::{DbConfig, RocksDbConfig};
 use crate::conversion::bytes_to_str;
 use actix_web::rt as actix_rt;
 
-
 use super::*;
 
 impl DbConfig {