From a926dbb256d1076217014c5b6c474efcbf720787 Mon Sep 17 00:00:00 2001 From: Guoteng Rao <3603304+grao1991@users.noreply.github.com> Date: Thu, 8 Jun 2023 17:46:24 -0700 Subject: [PATCH] [Storage][Sharding] Implement create_checkpoint for LedgerDb, and deprecate use_state_kv_db flag. --- aptos-node/src/storage.rs | 1 + config/src/config/storage_config.rs | 4 -- .../executor-benchmark/src/db_generator.rs | 15 ++++-- execution/executor-benchmark/src/lib.rs | 24 ++++++--- execution/executor-benchmark/src/main.rs | 8 +-- .../aptosdb/src/db_debugger/checkpoint/mod.rs | 4 +- .../db_debugger/examine/print_db_versions.rs | 4 +- .../aptosdb/src/db_debugger/truncate/mod.rs | 13 +++-- storage/aptosdb/src/ledger_db.rs | 50 +++++++++++++++++-- storage/aptosdb/src/lib.rs | 26 ++++------ storage/aptosdb/src/state_kv_db.rs | 2 +- storage/backup/backup-cli/src/utils/mod.rs | 3 -- testsuite/single_node_performance.py | 4 +- testsuite/smoke-test/src/genesis.rs | 2 +- testsuite/smoke-test/src/storage.rs | 8 +-- 15 files changed, 108 insertions(+), 60 deletions(-) diff --git a/aptos-node/src/storage.rs b/aptos-node/src/storage.rs index 5a71240ff61e7..84b44a7b3d82a 100644 --- a/aptos-node/src/storage.rs +++ b/aptos-node/src/storage.rs @@ -62,6 +62,7 @@ fn create_rocksdb_checkpoint_and_change_working_dir( AptosDB::create_checkpoint( &source_dir, &checkpoint_dir, + node_config.storage.rocksdb_configs.split_ledger_db, node_config .storage .rocksdb_configs diff --git a/config/src/config/storage_config.rs b/config/src/config/storage_config.rs index c9f29cd240ade..933fd1d940db4 100644 --- a/config/src/config/storage_config.rs +++ b/config/src/config/storage_config.rs @@ -65,9 +65,6 @@ pub struct RocksdbConfigs { pub ledger_db_config: RocksdbConfig, pub state_merkle_db_config: RocksdbConfig, // Note: Not ready for production use yet. - // TODO(grao): Deprecate this flag and use the split_ledger_db_to_individual_dbs below. - pub use_state_kv_db: bool, - // Note: Not ready for production use yet. pub use_sharded_state_merkle_db: bool, // Note: Not ready for production use yet. // TODO(grao): Add RocksdbConfig for individual DBs when necessary. @@ -81,7 +78,6 @@ impl Default for RocksdbConfigs { Self { ledger_db_config: RocksdbConfig::default(), state_merkle_db_config: RocksdbConfig::default(), - use_state_kv_db: false, use_sharded_state_merkle_db: false, split_ledger_db: false, state_kv_db_config: RocksdbConfig::default(), diff --git a/execution/executor-benchmark/src/db_generator.rs b/execution/executor-benchmark/src/db_generator.rs index 1faf1d8d1c8f3..1cd8c43a032af 100644 --- a/execution/executor-benchmark/src/db_generator.rs +++ b/execution/executor-benchmark/src/db_generator.rs @@ -26,7 +26,7 @@ pub fn create_db_with_accounts( db_dir: impl AsRef, storage_pruner_config: PrunerConfig, verify_sequence_numbers: bool, - use_state_kv_db: bool, + split_ledger_db: bool, use_sharded_state_merkle_db: bool, pipeline_config: PipelineConfig, ) where @@ -40,7 +40,7 @@ pub fn create_db_with_accounts( // create if not exists fs::create_dir_all(db_dir.as_ref()).unwrap(); - bootstrap_with_genesis(&db_dir, use_state_kv_db); + bootstrap_with_genesis(&db_dir, split_ledger_db, use_sharded_state_merkle_db); println!( "Finished empty DB creation, DB dir: {}. Creating accounts now...", @@ -55,18 +55,23 @@ pub fn create_db_with_accounts( &db_dir, storage_pruner_config, verify_sequence_numbers, - use_state_kv_db, + split_ledger_db, use_sharded_state_merkle_db, pipeline_config, ); } -fn bootstrap_with_genesis(db_dir: impl AsRef, use_state_kv_db: bool) { +fn bootstrap_with_genesis( + db_dir: impl AsRef, + split_ledger_db: bool, + use_sharded_state_merkle_db: bool, +) { let (config, _genesis_key) = aptos_genesis::test_utils::test_config(); let mut rocksdb_configs = RocksdbConfigs::default(); rocksdb_configs.state_merkle_db_config.max_open_files = -1; - rocksdb_configs.use_state_kv_db = use_state_kv_db; + rocksdb_configs.split_ledger_db = split_ledger_db; + rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db; let (_db, db_rw) = DbReaderWriter::wrap( AptosDB::open( &db_dir, diff --git a/execution/executor-benchmark/src/lib.rs b/execution/executor-benchmark/src/lib.rs index 061b88dd5328f..143a93d6a58df 100644 --- a/execution/executor-benchmark/src/lib.rs +++ b/execution/executor-benchmark/src/lib.rs @@ -73,6 +73,7 @@ where fn create_checkpoint( source_dir: impl AsRef, checkpoint_dir: impl AsRef, + split_ledger_db: bool, use_sharded_state_merkle_db: bool, ) { // Create rocksdb checkpoint. @@ -81,8 +82,13 @@ fn create_checkpoint( } std::fs::create_dir_all(checkpoint_dir.as_ref()).unwrap(); - AptosDB::create_checkpoint(source_dir, checkpoint_dir, use_sharded_state_merkle_db) - .expect("db checkpoint creation fails."); + AptosDB::create_checkpoint( + source_dir, + checkpoint_dir, + split_ledger_db, + use_sharded_state_merkle_db, + ) + .expect("db checkpoint creation fails."); } /// Runs the benchmark with given parameters. @@ -98,7 +104,7 @@ pub fn run_benchmark( checkpoint_dir: impl AsRef, verify_sequence_numbers: bool, pruner_config: PrunerConfig, - use_state_kv_db: bool, + split_ledger_db: bool, use_sharded_state_merkle_db: bool, pipeline_config: PipelineConfig, ) where @@ -107,13 +113,14 @@ pub fn run_benchmark( create_checkpoint( source_dir.as_ref(), checkpoint_dir.as_ref(), + split_ledger_db, use_sharded_state_merkle_db, ); let (mut config, genesis_key) = aptos_genesis::test_utils::test_config(); config.storage.dir = checkpoint_dir.as_ref().to_path_buf(); config.storage.storage_pruner_config = pruner_config; - config.storage.rocksdb_configs.use_state_kv_db = use_state_kv_db; + config.storage.rocksdb_configs.split_ledger_db = split_ledger_db; config.storage.rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db; let (db, executor) = init_db_and_executor::(&config); @@ -349,7 +356,7 @@ pub fn add_accounts( checkpoint_dir: impl AsRef, pruner_config: PrunerConfig, verify_sequence_numbers: bool, - use_state_kv_db: bool, + split_ledger_db: bool, use_sharded_state_merkle_db: bool, pipeline_config: PipelineConfig, ) where @@ -359,6 +366,7 @@ pub fn add_accounts( create_checkpoint( source_dir.as_ref(), checkpoint_dir.as_ref(), + split_ledger_db, use_sharded_state_merkle_db, ); add_accounts_impl::( @@ -369,7 +377,7 @@ pub fn add_accounts( checkpoint_dir, pruner_config, verify_sequence_numbers, - use_state_kv_db, + split_ledger_db, use_sharded_state_merkle_db, pipeline_config, ); @@ -383,7 +391,7 @@ fn add_accounts_impl( output_dir: impl AsRef, pruner_config: PrunerConfig, verify_sequence_numbers: bool, - use_state_kv_db: bool, + split_ledger_db: bool, use_sharded_state_merkle_db: bool, pipeline_config: PipelineConfig, ) where @@ -392,7 +400,7 @@ fn add_accounts_impl( let (mut config, genesis_key) = aptos_genesis::test_utils::test_config(); config.storage.dir = output_dir.as_ref().to_path_buf(); config.storage.storage_pruner_config = pruner_config; - config.storage.rocksdb_configs.use_state_kv_db = use_state_kv_db; + config.storage.rocksdb_configs.split_ledger_db = split_ledger_db; config.storage.rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db; let (db, executor) = init_db_and_executor::(&config); diff --git a/execution/executor-benchmark/src/main.rs b/execution/executor-benchmark/src/main.rs index 44494f3e48562..4fa2e9cc25373 100644 --- a/execution/executor-benchmark/src/main.rs +++ b/execution/executor-benchmark/src/main.rs @@ -124,7 +124,7 @@ struct Opt { pruner_opt: PrunerOpt, #[clap(long)] - use_state_kv_db: bool, + split_ledger_db: bool, #[clap(long)] use_sharded_state_merkle_db: bool, @@ -229,7 +229,7 @@ where data_dir, opt.pruner_opt.pruner_config(), opt.verify_sequence_numbers, - opt.use_state_kv_db, + opt.split_ledger_db, opt.use_sharded_state_merkle_db, opt.pipeline_opt.pipeline_config(), ); @@ -254,7 +254,7 @@ where checkpoint_dir, opt.verify_sequence_numbers, opt.pruner_opt.pruner_config(), - opt.use_state_kv_db, + opt.split_ledger_db, opt.use_sharded_state_merkle_db, opt.pipeline_opt.pipeline_config(), ); @@ -273,7 +273,7 @@ where checkpoint_dir, opt.pruner_opt.pruner_config(), opt.verify_sequence_numbers, - opt.use_state_kv_db, + opt.split_ledger_db, opt.use_sharded_state_merkle_db, opt.pipeline_opt.pipeline_config(), ); diff --git a/storage/aptosdb/src/db_debugger/checkpoint/mod.rs b/storage/aptosdb/src/db_debugger/checkpoint/mod.rs index e71618e31e3ad..d9cb4b03773b4 100644 --- a/storage/aptosdb/src/db_debugger/checkpoint/mod.rs +++ b/storage/aptosdb/src/db_debugger/checkpoint/mod.rs @@ -21,7 +21,7 @@ impl Cmd { ensure!(!self.output_dir.exists(), "Output dir already exists."); fs::create_dir_all(&self.output_dir)?; - // TODO(grao): Support sharded state merkle db here. - AptosDB::create_checkpoint(self.db_dir, self.output_dir, false) + // TODO(grao): Support sharded state merkle db and split_ledger_db here. + AptosDB::create_checkpoint(self.db_dir, self.output_dir, false, false) } } diff --git a/storage/aptosdb/src/db_debugger/examine/print_db_versions.rs b/storage/aptosdb/src/db_debugger/examine/print_db_versions.rs index 10a8d094616e3..f3384d70f6c3b 100644 --- a/storage/aptosdb/src/db_debugger/examine/print_db_versions.rs +++ b/storage/aptosdb/src/db_debugger/examine/print_db_versions.rs @@ -33,13 +33,13 @@ pub struct Cmd { db_dir: PathBuf, #[clap(long)] - use_state_kv_db: bool, + split_ledger_db: bool, } impl Cmd { pub fn run(self) -> Result<()> { let rocksdb_config = RocksdbConfigs { - use_state_kv_db: self.use_state_kv_db, + split_ledger_db: self.split_ledger_db, ..Default::default() }; let (ledger_db, state_merkle_db, state_kv_db) = AptosDB::open_dbs( diff --git a/storage/aptosdb/src/db_debugger/truncate/mod.rs b/storage/aptosdb/src/db_debugger/truncate/mod.rs index ce9c0f309113d..04c953abfae6d 100644 --- a/storage/aptosdb/src/db_debugger/truncate/mod.rs +++ b/storage/aptosdb/src/db_debugger/truncate/mod.rs @@ -47,7 +47,7 @@ pub struct Cmd { opt_out_backup_checkpoint: bool, #[clap(long)] - use_state_kv_db: bool, + split_ledger_db: bool, } impl Cmd { @@ -61,14 +61,19 @@ impl Cmd { println!("Creating backup at: {:?}", &backup_checkpoint_dir); fs::create_dir_all(&backup_checkpoint_dir)?; // TODO(grao): Support sharded state merkle db here. - AptosDB::create_checkpoint(&self.db_dir, backup_checkpoint_dir, false)?; + AptosDB::create_checkpoint( + &self.db_dir, + backup_checkpoint_dir, + self.split_ledger_db, + false, + )?; println!("Done!"); } else { println!("Opted out backup creation!."); } let rocksdb_config = RocksdbConfigs { - use_state_kv_db: self.use_state_kv_db, + split_ledger_db: self.split_ledger_db, ..Default::default() }; let (ledger_db, state_merkle_db, state_kv_db) = AptosDB::open_dbs( @@ -242,7 +247,7 @@ mod test { ledger_db_batch_size: 15, opt_out_backup_checkpoint: true, backup_checkpoint_dir: None, - use_state_kv_db: false, + split_ledger_db: false, }; cmd.run().unwrap(); diff --git a/storage/aptosdb/src/ledger_db.rs b/storage/aptosdb/src/ledger_db.rs index 9438d4c7bfda7..a2c46fcf67486 100644 --- a/storage/aptosdb/src/ledger_db.rs +++ b/storage/aptosdb/src/ledger_db.rs @@ -133,11 +133,53 @@ impl LedgerDb { } pub(crate) fn create_checkpoint( - _db_root_path: impl AsRef, - _cp_root_path: impl AsRef, + db_root_path: impl AsRef, + cp_root_path: impl AsRef, + split_ledger_db: bool, ) -> Result<()> { - // TODO(grao): Implement this function. - todo!() + let rocksdb_configs = RocksdbConfigs { + split_ledger_db, + ..Default::default() + }; + let ledger_db = Self::new(db_root_path, rocksdb_configs, /*readonly=*/ false)?; + let cp_ledger_db_folder = cp_root_path.as_ref().join(LEDGER_DB_FOLDER_NAME); + + info!( + split_ledger_db = split_ledger_db, + "Creating ledger_db checkpoint at: {cp_ledger_db_folder:?}" + ); + + std::fs::remove_dir_all(&cp_ledger_db_folder).unwrap_or(()); + if split_ledger_db { + std::fs::create_dir_all(&cp_ledger_db_folder).unwrap_or(()); + } + + ledger_db + .metadata_db() + .create_checkpoint(Self::metadata_db_path( + cp_root_path.as_ref(), + split_ledger_db, + ))?; + + if split_ledger_db { + ledger_db + .event_db() + .create_checkpoint(cp_ledger_db_folder.join(EVENT_DB_NAME))?; + ledger_db + .transaction_accumulator_db() + .create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_ACCUMULATOR_DB_NAME))?; + ledger_db + .transaction_db() + .create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_DB_NAME))?; + ledger_db + .transaction_info_db() + .create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_INFO_DB_NAME))?; + ledger_db + .write_set_db() + .create_checkpoint(cp_ledger_db_folder.join(WRITE_SET_DB_NAME))?; + } + + Ok(()) } pub(crate) fn write_pruner_progress(&self, version: Version) -> Result<()> { diff --git a/storage/aptosdb/src/lib.rs b/storage/aptosdb/src/lib.rs index d43f1741d0e91..50885c6c22a6b 100644 --- a/storage/aptosdb/src/lib.rs +++ b/storage/aptosdb/src/lib.rs @@ -703,27 +703,21 @@ impl AptosDB { pub fn create_checkpoint( db_path: impl AsRef, cp_path: impl AsRef, + use_split_ledger_db: bool, use_sharded_state_merkle_db: bool, ) -> Result<()> { let start = Instant::now(); - let ledger_db_path = db_path.as_ref().join(LEDGER_DB_NAME); - let ledger_cp_path = cp_path.as_ref().join(LEDGER_DB_NAME); - info!("Creating ledger_db checkpoint at: {ledger_cp_path:?}"); - - std::fs::remove_dir_all(&ledger_cp_path).unwrap_or(()); - - // Weird enough, checkpoint doesn't work with readonly or secondary mode (gets stuck). - // https://github.com/facebook/rocksdb/issues/11167 - let ledger_db = aptos_schemadb::DB::open( - ledger_db_path, - LEDGER_DB_NAME, - ledger_db_column_families(), - &aptos_schemadb::Options::default(), - )?; - ledger_db.create_checkpoint(ledger_cp_path)?; + info!( + use_split_ledger_db = use_split_ledger_db, + use_sharded_state_merkle_db = use_sharded_state_merkle_db, + "Creating checkpoint for AptosDB." + ); - StateKvDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref())?; + LedgerDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref(), use_split_ledger_db)?; + if use_split_ledger_db { + StateKvDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref())?; + } StateMerkleDb::create_checkpoint( db_path.as_ref(), cp_path.as_ref(), diff --git a/storage/aptosdb/src/state_kv_db.rs b/storage/aptosdb/src/state_kv_db.rs index fb20066ee267b..d37a385343168 100644 --- a/storage/aptosdb/src/state_kv_db.rs +++ b/storage/aptosdb/src/state_kv_db.rs @@ -38,7 +38,7 @@ impl StateKvDb { readonly: bool, ledger_db: Arc, ) -> Result { - if !rocksdb_configs.use_state_kv_db { + if !rocksdb_configs.split_ledger_db { info!("State K/V DB is not enabled!"); return Ok(Self { state_kv_metadata_db: Arc::clone(&ledger_db), diff --git a/storage/backup/backup-cli/src/utils/mod.rs b/storage/backup/backup-cli/src/utils/mod.rs index 3067cf0f6c825..948b05f9be0f4 100644 --- a/storage/backup/backup-cli/src/utils/mod.rs +++ b/storage/backup/backup-cli/src/utils/mod.rs @@ -69,8 +69,6 @@ pub struct RocksdbOpt { #[clap(long, hidden(true))] split_ledger_db: bool, #[clap(long, hidden(true))] - use_state_kv_db: bool, - #[clap(long, hidden(true))] use_sharded_state_merkle_db: bool, #[clap(long, hidden(true), default_value = "5000")] state_kv_db_max_open_files: i32, @@ -100,7 +98,6 @@ impl From for RocksdbConfigs { ..Default::default() }, split_ledger_db: opt.split_ledger_db, - use_state_kv_db: opt.use_state_kv_db, use_sharded_state_merkle_db: opt.use_sharded_state_merkle_db, state_kv_db_config: RocksdbConfig { max_open_files: opt.state_kv_db_max_open_files, diff --git a/testsuite/single_node_performance.py b/testsuite/single_node_performance.py index 301af0a791232..deeea5f6d4e17 100755 --- a/testsuite/single_node_performance.py +++ b/testsuite/single_node_performance.py @@ -222,7 +222,7 @@ def print_table( warnings = [] with tempfile.TemporaryDirectory() as tmpdirname: - create_db_command = f"cargo run {BUILD_FLAG} -- --block-size {BLOCK_SIZE} --concurrency-level {CONCURRENCY_LEVEL} --use-state-kv-db --use-sharded-state-merkle-db create-db --data-dir {tmpdirname}/db --num-accounts {NUM_ACCOUNTS}" + create_db_command = f"cargo run {BUILD_FLAG} -- --block-size {BLOCK_SIZE} --concurrency-level {CONCURRENCY_LEVEL} --split-ledger-db --use-sharded-state-merkle-db create-db --data-dir {tmpdirname}/db --num-accounts {NUM_ACCOUNTS}" output = execute_command(create_db_command) results = [] @@ -237,7 +237,7 @@ def print_table( executor_type = "native" if use_native_executor else "VM" use_native_executor_str = "--use-native-executor" if use_native_executor else "" - common_command_suffix = f"{use_native_executor_str} --generate-then-execute --transactions-per-sender 1 --block-size {cur_block_size} --use-state-kv-db --use-sharded-state-merkle-db run-executor --transaction-type {transaction_type} --module-working-set-size {module_working_set_size} --main-signer-accounts {MAIN_SIGNER_ACCOUNTS} --additional-dst-pool-accounts {ADDITIONAL_DST_POOL_ACCOUNTS} --data-dir {tmpdirname}/db --checkpoint-dir {tmpdirname}/cp" + common_command_suffix = f"{use_native_executor_str} --generate-then-execute --transactions-per-sender 1 --block-size {cur_block_size} --split-ledger-db --use-sharded-state-merkle-db run-executor --transaction-type {transaction_type} --module-working-set-size {module_working_set_size} --main-signer-accounts {MAIN_SIGNER_ACCOUNTS} --additional-dst-pool-accounts {ADDITIONAL_DST_POOL_ACCOUNTS} --data-dir {tmpdirname}/db --checkpoint-dir {tmpdirname}/cp" concurrency_level_results = {} diff --git a/testsuite/smoke-test/src/genesis.rs b/testsuite/smoke-test/src/genesis.rs index abf21ef8bb505..75ae619ace1bd 100644 --- a/testsuite/smoke-test/src/genesis.rs +++ b/testsuite/smoke-test/src/genesis.rs @@ -271,7 +271,7 @@ async fn test_genesis_transaction_flow() { backup_path.path(), db_dir.as_path(), &[waypoint], - node.config().storage.rocksdb_configs.use_state_kv_db, + node.config().storage.rocksdb_configs.split_ledger_db, None, ); diff --git a/testsuite/smoke-test/src/storage.rs b/testsuite/smoke-test/src/storage.rs index 8c1cfc99d684c..a40490037e639 100644 --- a/testsuite/smoke-test/src/storage.rs +++ b/testsuite/smoke-test/src/storage.rs @@ -131,7 +131,7 @@ async fn test_db_restore() { backup_path.path(), db_dir.as_path(), &[], - node0_config.storage.rocksdb_configs.use_state_kv_db, + node0_config.storage.rocksdb_configs.split_ledger_db, None, ); @@ -408,7 +408,7 @@ pub(crate) fn db_restore( backup_path: &Path, db_path: &Path, trusted_waypoints: &[Waypoint], - use_state_kv_db: bool, + split_ledger_db: bool, target_verion: Option, /* target version should be same as epoch ending version to start a node */ ) { let now = Instant::now(); @@ -424,8 +424,8 @@ pub(crate) fn db_restore( cmd.arg(&w.to_string()); }); - if use_state_kv_db { - cmd.arg("--use-state-kv-db"); + if split_ledger_db { + cmd.arg("--split-ledger-db"); } if let Some(version) = target_verion { cmd.arg("--target-version");