feat: enable config rocksdb by options

common/config-parser/src/types.rs

@@ -11,6 +11,7 @@ pub const DEFAULT_BROADCAST_TXS_SIZE: usize = 200;
 pub const DEFAULT_BROADCAST_TXS_INTERVAL: u64 = 200; // milliseconds
 pub const DEFAULT_OVERLORD_GAP: usize = 5;
 pub const DEFAULT_SYNC_TXS_CHUNK_SIZE: usize = 5000;
+pub const DEFAULT_CACHE_SIZE: usize = 128 << 20;
 
 #[derive(Clone, Debug, Deserialize)]
 pub struct ConfigApi {
@@ -100,14 +101,25 @@ pub struct ConfigExecutor {
     pub triedb_cache_size: usize,
 }
 
+fn default_cache_size() -> usize {
+    DEFAULT_CACHE_SIZE
+}
+
 #[derive(Clone, Debug, Deserialize)]
 pub struct ConfigRocksDB {
     pub max_open_files: i32,
+    #[serde(default = "default_cache_size")]
+    pub cache_size:     usize,
+    pub options_file:   Option<PathBuf>,
 }
 
 impl Default for ConfigRocksDB {
     fn default() -> Self {
-        Self { max_open_files: 64 }
+        Self {
+            max_open_files: 64,
+            cache_size:     default_cache_size(),
+            options_file:   None,
+        }
     }
 }
 

core/cli/src/lib.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use clap::{crate_version, Arg, ArgMatches, Command};
 
 use common_config_parser::{parse_file, types::Config};
@@ -35,8 +37,14 @@ impl AxonCli {
     }
 
     pub fn start(&self) {
-        let config: Config =
-            parse_file(self.matches.value_of("config_path").unwrap(), false).unwrap();
+        let config_path = self.matches.value_of("config_path").unwrap();
+        let path = Path::new(config_path).parent().unwrap();
+
+        let mut config: Config = parse_file(config_path, false).unwrap();
+
+        if let Some(ref mut f) = config.rocksdb.options_file {
+            *f = path.join(&f)
+        }
         let genesis: RichBlock =
             parse_file(self.matches.value_of("genesis_path").unwrap(), true).unwrap();
 

core/executor/Cargo.toml

@@ -24,6 +24,7 @@ rug = "1.15"
 sha2 = "0.10"
 
 common-apm = { path = "../../common/apm" }
+common-config-parser = { path = "../../common/config-parser" }
 common-crypto = { path = "../../common/crypto" }
 common-merkle = { path = "../../common/merkle" }
 core-interoperation = { path = "../interoperation" }

core/executor/benches/bench_transfer.rs

@@ -34,9 +34,9 @@ struct BenchAdapter {
 impl BenchAdapter {
     fn new() -> Self {
         BenchAdapter {
-            trie_db: Arc::new(RocksTrieDB::new(STATE_PATH, 1000, 1000).unwrap()),
+            trie_db: Arc::new(RocksTrieDB::new(STATE_PATH, Default::default(), 1000).unwrap()),
             storage: Arc::new(ImplStorage::new(Arc::new(
-                RocksAdapter::new(DATA_PATH, 1000).unwrap(),
+                RocksAdapter::new(DATA_PATH, Default::default()).unwrap(),
             ))),
         }
     }

core/executor/src/adapter/trie_db.rs

@@ -3,10 +3,11 @@ use std::{collections::HashSet, fs, io, path::Path, sync::Arc};
 use dashmap::DashMap;
 use rand::{rngs::SmallRng, Rng, SeedableRng};
 use rocksdb::ops::{Get, Open, Put, WriteOps};
-use rocksdb::{Options, WriteBatch, DB};
+use rocksdb::{FullOptions, Options, WriteBatch, DB};
 
 use common_apm::metrics::storage::{on_storage_get_state, on_storage_put_state};
 use common_apm::Instant;
+use common_config_parser::types::ConfigRocksDB;
 use protocol::{Display, From, ProtocolError, ProtocolErrorKind, ProtocolResult};
 
 // 49999 is the largest prime number within 50000.
@@ -21,17 +22,30 @@ pub struct RocksTrieDB {
 impl RocksTrieDB {
     pub fn new<P: AsRef<Path>>(
         path: P,
-        max_open_files: i32,
+        config: ConfigRocksDB,
         cache_size: usize,
     ) -> ProtocolResult<Self> {
         if !path.as_ref().is_dir() {
             fs::create_dir_all(&path).map_err(RocksTrieDBError::CreateDB)?;
         }
 
-        let mut opts = Options::default();
+        let mut opts = if let Some(ref file) = config.options_file {
+            let cache_size = match config.cache_size {
+                0 => None,
+                size => Some(size),
+            };
+
+            let full_opts = FullOptions::load_from_file(file, cache_size, false)
+                .map_err(RocksTrieDBError::from)?;
+
+            let FullOptions { db_opts, .. } = full_opts;
+            db_opts
+        } else {
+            Options::default()
+        };
         opts.create_if_missing(true);
         opts.create_missing_column_families(true);
-        opts.set_max_open_files(max_open_files);
+        opts.set_max_open_files(config.max_open_files);
 
         let db = DB::open(&opts, path).map_err(RocksTrieDBError::from)?;
 
@@ -247,7 +261,7 @@ mod tests {
         let val_2 = rand_bytes(256);
 
         let dir = tempfile::tempdir().unwrap();
-        let trie = RocksTrieDB::new(dir.path(), 1024, 100).unwrap();
+        let trie = RocksTrieDB::new(dir.path(), Default::default(), 100).unwrap();
 
         trie.insert(key_1.clone(), val_1.clone()).unwrap();
         trie.insert(key_2.clone(), val_2.clone()).unwrap();
@@ -274,7 +288,7 @@ mod tests {
         let val_2 = rand_bytes(256);
 
         let dir = tempfile::tempdir().unwrap();
-        let trie = RocksTrieDB::new(dir.path(), 1024, 100).unwrap();
+        let trie = RocksTrieDB::new(dir.path(), Default::default(), 100).unwrap();
 
         trie.insert(key_1.clone(), val_1.clone()).unwrap();
         trie.insert(key_2.clone(), val_2.clone()).unwrap();

core/executor/src/debugger/mod.rs

@@ -28,10 +28,10 @@ impl EvmDebugger {
     pub fn new(distribute_address: H160, distribute_amount: U256, db_path: &str) -> Self {
         let mut db_data_path = db_path.to_string();
         db_data_path.push_str("/data");
-        let rocks_adapter = Arc::new(RocksAdapter::new(db_data_path, 1024).unwrap());
+        let rocks_adapter = Arc::new(RocksAdapter::new(db_data_path, Default::default()).unwrap());
         let mut db_state_path = db_path.to_string();
         db_state_path.push_str("/state");
-        let trie = Arc::new(RocksTrieDB::new(db_state_path, 1024, 1000).unwrap());
+        let trie = Arc::new(RocksTrieDB::new(db_state_path, Default::default(), 1000).unwrap());
 
         let mut mpt = MPTTrie::new(Arc::clone(&trie));
 

core/metadata/src/tests/mod.rs

@@ -35,9 +35,13 @@ struct TestHandle {
 impl TestHandle {
     pub async fn new(salt: u64) -> Self {
         let path = "./free-space/".to_string();
-        let storage_adapter =
-            RocksAdapter::new(path.clone() + &salt.to_string() + "/rocks", 1024).unwrap();
-        let trie_db = RocksTrieDB::new(path + &salt.to_string() + "/trie", 1024, 50).unwrap();
+        let storage_adapter = RocksAdapter::new(
+            path.clone() + &salt.to_string() + "/rocks",
+            Default::default(),
+        )
+        .unwrap();
+        let trie_db =
+            RocksTrieDB::new(path + &salt.to_string() + "/trie", Default::default(), 50).unwrap();
 
         let mut handle = TestHandle {
             storage:    Arc::new(ImplStorage::new(Arc::new(storage_adapter))),

core/run/src/lib.rs

@@ -94,10 +94,7 @@ impl Axon {
     pub async fn create_genesis(&mut self) -> ProtocolResult<()> {
         // Init Block db
         let path_block = self.config.data_path_for_block();
-        let rocks_adapter = Arc::new(RocksAdapter::new(
-            path_block,
-            self.config.rocksdb.max_open_files,
-        )?);
+        let rocks_adapter = Arc::new(RocksAdapter::new(path_block, self.config.rocksdb.clone())?);
         let storage = Arc::new(ImplStorage::new(rocks_adapter));
 
         match storage.get_latest_block(Context::new()).await {
@@ -116,7 +113,7 @@ impl Axon {
         let path_state = self.config.data_path_for_state();
         let trie_db = Arc::new(RocksTrieDB::new(
             path_state,
-            self.config.rocksdb.max_open_files,
+            self.config.rocksdb.clone(),
             self.config.executor.triedb_cache_size,
         )?);
         let mut mpt = MPTTrie::new(Arc::clone(&trie_db));
@@ -186,7 +183,7 @@ impl Axon {
 
         let rocks_adapter = Arc::new(RocksAdapter::new(
             path_block.clone(),
-            config.rocksdb.max_open_files,
+            config.rocksdb.clone(),
         )?);
         let storage = Arc::new(ImplStorage::new(rocks_adapter));
 
@@ -228,7 +225,7 @@ impl Axon {
         let path_state = config.data_path_for_state();
         let trie_db = Arc::new(RocksTrieDB::new(
             path_state,
-            config.rocksdb.max_open_files,
+            config.rocksdb.clone(),
             config.executor.triedb_cache_size,
         )?);
 

core/storage/Cargo.toml

@@ -15,6 +15,7 @@ rocksdb = { version = "0.16", package = "ckb-rocksdb" }
 
 common-apm = { path = "../../common/apm" }
 common-apm-derive = { path = "../../common/apm-derive" }
+common-config-parser = { path = "../../common/config-parser" }
 protocol = { path = "../../protocol", package = "axon-protocol" }
 
 [dev-dependencies]

core/storage/src/adapter/rocks.rs

@@ -5,10 +5,13 @@ use std::sync::Arc;
 use std::{fs, io};
 
 use rocksdb::ops::{DeleteCF, GetCF, GetColumnFamilys, IterateCF, OpenCF, PutCF, WriteOps};
-use rocksdb::{ColumnFamily, DBIterator, Options, WriteBatch, DB};
+use rocksdb::{
+    ColumnFamily, ColumnFamilyDescriptor, DBIterator, FullOptions, Options, WriteBatch, DB,
+};
 
 use common_apm::metrics::storage::on_storage_put_cf;
 use common_apm::Instant;
+use common_config_parser::types::ConfigRocksDB;
 use protocol::codec::{hex_encode, ProtocolCodec};
 use protocol::traits::{
     IntoIteratorByRef, StorageAdapter, StorageBatchModify, StorageCategory, StorageIterator,
@@ -24,16 +27,10 @@ pub struct RocksAdapter {
 }
 
 impl RocksAdapter {
-    pub fn new<P: AsRef<Path>>(path: P, max_open_files: i32) -> ProtocolResult<Self> {
+    pub fn new<P: AsRef<Path>>(path: P, config: ConfigRocksDB) -> ProtocolResult<Self> {
         if !path.as_ref().is_dir() {
             fs::create_dir_all(&path).map_err(RocksAdapterError::CreateDB)?;
         }
-
-        let mut opts = Options::default();
-        opts.create_if_missing(true);
-        opts.create_missing_column_families(true);
-        opts.set_max_open_files(max_open_files);
-
         let categories = [
             map_category(StorageCategory::Block),
             map_category(StorageCategory::BlockHeader),
@@ -44,7 +41,38 @@ impl RocksAdapter {
             map_category(StorageCategory::Code),
         ];
 
-        let db = DB::open_cf(&opts, path, categories.iter()).map_err(RocksAdapterError::from)?;
+        let (mut opts, cf_descriptors) = if let Some(ref file) = config.options_file {
+            let cache_size = match config.cache_size {
+                0 => None,
+                size => Some(size),
+            };
+
+            let mut full_opts = FullOptions::load_from_file(file, cache_size, false)
+                .map_err(RocksAdapterError::from)?;
+
+            full_opts
+                .complete_column_families(&categories, false)
+                .map_err(RocksAdapterError::from)?;
+            let FullOptions {
+                db_opts,
+                cf_descriptors,
+            } = full_opts;
+            (db_opts, cf_descriptors)
+        } else {
+            let opts = Options::default();
+            let cf_descriptors: Vec<_> = categories
+                .into_iter()
+                .map(|c| ColumnFamilyDescriptor::new(c, Options::default()))
+                .collect();
+            (opts, cf_descriptors)
+        };
+
+        opts.create_if_missing(true);
+        opts.create_missing_column_families(true);
+        opts.set_max_open_files(config.max_open_files);
+
+        let db = DB::open_cf_descriptors(&opts, path, cf_descriptors)
+            .map_err(RocksAdapterError::from)?;
 
         Ok(RocksAdapter { db: Arc::new(db) })
     }

core/storage/src/tests/adapter.rs

@@ -8,19 +8,25 @@ use crate::{CommonHashKey, TransactionSchema};
 #[test]
 fn test_adapter_insert() {
     adapter_insert_test(MemoryAdapter::new());
-    adapter_insert_test(RocksAdapter::new("rocksdb/test_adapter_insert", 64).unwrap())
+    adapter_insert_test(
+        RocksAdapter::new("rocksdb/test_adapter_insert", Default::default()).unwrap(),
+    )
 }
 
 #[test]
 fn test_adapter_batch_modify() {
     adapter_batch_modify_test(MemoryAdapter::new());
-    adapter_batch_modify_test(RocksAdapter::new("rocksdb/test_adapter_batch_modify", 64).unwrap())
+    adapter_batch_modify_test(
+        RocksAdapter::new("rocksdb/test_adapter_batch_modify", Default::default()).unwrap(),
+    )
 }
 
 #[test]
 fn test_adapter_remove() {
     adapter_remove_test(MemoryAdapter::new());
-    adapter_remove_test(RocksAdapter::new("rocksdb/test_adapter_remove", 64).unwrap())
+    adapter_remove_test(
+        RocksAdapter::new("rocksdb/test_adapter_remove", Default::default()).unwrap(),
+    )
 }
 
 fn adapter_insert_test(db: impl StorageAdapter) {

devtools/chain/config.toml

@@ -54,6 +54,11 @@ metrics = true
 
 [rocksdb]
 max_open_files = 64
+cache_size = 134217728
+
+# Provide an options file to tune RocksDB for your workload and your system configuration.
+# More details can be found in [the official tuning guide](https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide).
+options_file = "default.db-options"
 
 [jaeger]
 service_name = "axon"
@@ -80,4 +85,4 @@ checkpoint_type_hash = "0xf32ec1e3274c27ac4cc018c92efec443847e9b78ba8cbedca36f0f
 [interoperability_extension]
 blockchain_extension_transaction_hashes = [
     {name = "cardano", id = 2, tx_hash = "0xb1af175009413bf9670dffb7b120f0eca52896a9798bda123df9b25ff7d8f721"}
-]
+]

devtools/chain/default.db-options

@@ -0,0 +1,23 @@
+# This is a RocksDB option file.
+#
+# For detailed file format spec, please refer to the official documents
+# in https://rocksdb.org/docs/
+#
+
+[DBOptions]
+bytes_per_sync=1048576
+max_background_compactions=4
+max_background_flushes=2
+max_total_wal_size=134217728
+keep_log_file_num=32
+
+[CFOptions "default"]
+level_compaction_dynamic_level_bytes=true
+write_buffer_size=8388608
+min_write_buffer_number_to_merge=1
+max_write_buffer_number=2
+max_write_buffer_size_to_maintain=-1
+
+[TableOptions/BlockBasedTable "default"]
+cache_index_and_filter_blocks=true
+pin_l0_filter_and_index_blocks_in_cache=true