diff --git a/README.md b/README.md index 972fbed..e984a06 100644 --- a/README.md +++ b/README.md @@ -94,9 +94,6 @@ $ cargo test -- --test-threads=1 ### TODO - - [X] actix-web 3.0 - fixed mem leak - - [X] channel for expire - - [ ] support for more rocksDb options in config (bloom, block cache..) - [ ] fix running test in parallel (TODO per test db root path) - [ ] db iterator for on startup init - [ ] test compaction diff --git a/config/db_config.toml b/config/db_config.toml index 78f02cb..23ee29f 100644 --- a/config/db_config.toml +++ b/config/db_config.toml @@ -13,4 +13,11 @@ level_zero_stop_writes_trigger = 24 level_zero_slowdown_writes_trigger = 20 compaction_style = "Level" #block based +use_cache = true +cache_defaults = true +cache_size = 100000 +block_cache = 10000 +bloom_filter_bits_per_key = 8 +bloom_filter_block_based = true +index_type = "HashSearch" diff --git a/src/config.rs b/src/config.rs index 807046c..9f5a770 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,5 @@ use confy::ConfyError; -use rocksdb::{DBCompactionStyle, Options}; +use rocksdb::{BlockBasedIndexType, BlockBasedOptions, Cache, DBCompactionStyle, Options}; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug)] @@ -68,6 +68,13 @@ pub struct RocksDbConfig { level_zero_stop_writes_trigger: i32, level_zero_slowdown_writes_trigger: i32, compaction_style: String, + use_cache: bool, + cache_defaults: bool, + cache_size: usize, + block_cache: usize, + bloom_filter_bits_per_key: i32, + bloom_filter_block_based: bool, + index_type: String, } impl Default for RocksDbConfig { @@ -87,6 +94,13 @@ impl Default for RocksDbConfig { level_zero_stop_writes_trigger: 24, level_zero_slowdown_writes_trigger: 24, compaction_style: "Level".to_string(), + use_cache: true, + cache_defaults: true, + cache_size: 100_000, + block_cache: 10_000, + bloom_filter_bits_per_key: 8, + bloom_filter_block_based: true, + index_type: "HashSearch".to_string(), } } } @@ -117,11 +131,30 @@ impl RocksDbConfig { opts.set_level_zero_slowdown_writes_trigger(self.level_zero_slowdown_writes_trigger); opts.set_compaction_style(get_compaction_style(&self.compaction_style)); opts.create_if_missing(true); + opts.set_block_based_table_factory(&block_based_opts(self)); opts } } +fn block_based_opts(cfg: &RocksDbConfig) -> BlockBasedOptions { + let mut opts = BlockBasedOptions::default(); + if cfg.cache_defaults { + opts + } else if !cfg.use_cache { + opts.disable_cache(); + opts + } else { + opts.set_block_cache( + &Cache::new_lru_cache(cfg.cache_size).expect("Could not create Cache"), + ); + opts.set_block_size(cfg.block_cache); + opts.set_bloom_filter(cfg.bloom_filter_bits_per_key, cfg.bloom_filter_block_based); + opts.set_index_type(get_index_type(&cfg.index_type)); + opts + } +} + pub fn load_db_config(cfg_path: &str) -> Result { let rocks_cfg = confy::load_path(format!("{}/db_config.toml", cfg_path))?; Ok(DbConfig::new(rocks_cfg)) @@ -149,3 +182,18 @@ fn get_compaction_style(s: &str) -> DBCompactionStyle { } } } + +fn get_index_type(s: &str) -> BlockBasedIndexType { + match s.to_lowercase().as_str() { + "hashsearch" => BlockBasedIndexType::HashSearch, + "binarysearch" => BlockBasedIndexType::BinarySearch, + "twolevelindexsearch" => BlockBasedIndexType::TwoLevelIndexSearch, + _ => { + error!( + "Unknown block based index type {} - fallback to default {}", + &s, "HashSearch" + ); + BlockBasedIndexType::HashSearch + } + } +} diff --git a/src/db.rs b/src/db.rs index 0ef0b5c..91ae26a 100644 --- a/src/db.rs +++ b/src/db.rs @@ -149,7 +149,6 @@ impl DbManager { .name("async-expire-thread".into()) .spawn(move || { for boxed in rx { - info!("expiring key in thead {:?}", thread::current()); boxed.invoke() } }) diff --git a/src/integration_tests.rs b/src/integration_tests.rs index aec5ac0..6fa6d55 100644 --- a/src/integration_tests.rs +++ b/src/integration_tests.rs @@ -9,7 +9,6 @@ use crate::config::{DbConfig, RocksDbConfig}; use crate::conversion::bytes_to_str; use actix_web::rt as actix_rt; - use super::*; impl DbConfig {