[Storage][Sharding] Implement create_checkpoint for LedgerDb, and dep…

…recate use_state_kv_db flag. (#8602)

aptos-node/src/storage.rs

@@ -62,6 +62,7 @@ fn create_rocksdb_checkpoint_and_change_working_dir(
     AptosDB::create_checkpoint(
         &source_dir,
         &checkpoint_dir,
+        node_config.storage.rocksdb_configs.split_ledger_db,
         node_config
             .storage
             .rocksdb_configs

config/src/config/storage_config.rs

@@ -65,9 +65,6 @@ pub struct RocksdbConfigs {
     pub ledger_db_config: RocksdbConfig,
     pub state_merkle_db_config: RocksdbConfig,
     // Note: Not ready for production use yet.
-    // TODO(grao): Deprecate this flag and use the split_ledger_db_to_individual_dbs below.
-    pub use_state_kv_db: bool,
-    // Note: Not ready for production use yet.
     pub use_sharded_state_merkle_db: bool,
     // Note: Not ready for production use yet.
     // TODO(grao): Add RocksdbConfig for individual DBs when necessary.
@@ -81,7 +78,6 @@ impl Default for RocksdbConfigs {
         Self {
             ledger_db_config: RocksdbConfig::default(),
             state_merkle_db_config: RocksdbConfig::default(),
-            use_state_kv_db: false,
             use_sharded_state_merkle_db: false,
             split_ledger_db: false,
             state_kv_db_config: RocksdbConfig::default(),

execution/executor-benchmark/src/db_generator.rs

@@ -26,7 +26,7 @@ pub fn create_db_with_accounts<V>(
     db_dir: impl AsRef<Path>,
     storage_pruner_config: PrunerConfig,
     verify_sequence_numbers: bool,
-    use_state_kv_db: bool,
+    split_ledger_db: bool,
     use_sharded_state_merkle_db: bool,
     pipeline_config: PipelineConfig,
 ) where
@@ -40,7 +40,7 @@ pub fn create_db_with_accounts<V>(
     // create if not exists
     fs::create_dir_all(db_dir.as_ref()).unwrap();
 
-    bootstrap_with_genesis(&db_dir, use_state_kv_db);
+    bootstrap_with_genesis(&db_dir, split_ledger_db, use_sharded_state_merkle_db);
 
     println!(
         "Finished empty DB creation, DB dir: {}. Creating accounts now...",
@@ -55,18 +55,23 @@ pub fn create_db_with_accounts<V>(
         &db_dir,
         storage_pruner_config,
         verify_sequence_numbers,
-        use_state_kv_db,
+        split_ledger_db,
         use_sharded_state_merkle_db,
         pipeline_config,
     );
 }
 
-fn bootstrap_with_genesis(db_dir: impl AsRef<Path>, use_state_kv_db: bool) {
+fn bootstrap_with_genesis(
+    db_dir: impl AsRef<Path>,
+    split_ledger_db: bool,
+    use_sharded_state_merkle_db: bool,
+) {
     let (config, _genesis_key) = aptos_genesis::test_utils::test_config();
 
     let mut rocksdb_configs = RocksdbConfigs::default();
     rocksdb_configs.state_merkle_db_config.max_open_files = -1;
-    rocksdb_configs.use_state_kv_db = use_state_kv_db;
+    rocksdb_configs.split_ledger_db = split_ledger_db;
+    rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db;
     let (_db, db_rw) = DbReaderWriter::wrap(
         AptosDB::open(
             &db_dir,

execution/executor-benchmark/src/lib.rs

@@ -73,6 +73,7 @@ where
 fn create_checkpoint(
     source_dir: impl AsRef<Path>,
     checkpoint_dir: impl AsRef<Path>,
+    split_ledger_db: bool,
     use_sharded_state_merkle_db: bool,
 ) {
     // Create rocksdb checkpoint.
@@ -81,8 +82,13 @@ fn create_checkpoint(
     }
     std::fs::create_dir_all(checkpoint_dir.as_ref()).unwrap();
 
-    AptosDB::create_checkpoint(source_dir, checkpoint_dir, use_sharded_state_merkle_db)
-        .expect("db checkpoint creation fails.");
+    AptosDB::create_checkpoint(
+        source_dir,
+        checkpoint_dir,
+        split_ledger_db,
+        use_sharded_state_merkle_db,
+    )
+    .expect("db checkpoint creation fails.");
 }
 
 /// Runs the benchmark with given parameters.
@@ -98,7 +104,7 @@ pub fn run_benchmark<V>(
     checkpoint_dir: impl AsRef<Path>,
     verify_sequence_numbers: bool,
     pruner_config: PrunerConfig,
-    use_state_kv_db: bool,
+    split_ledger_db: bool,
     use_sharded_state_merkle_db: bool,
     pipeline_config: PipelineConfig,
 ) where
@@ -107,13 +113,14 @@ pub fn run_benchmark<V>(
     create_checkpoint(
         source_dir.as_ref(),
         checkpoint_dir.as_ref(),
+        split_ledger_db,
         use_sharded_state_merkle_db,
     );
 
     let (mut config, genesis_key) = aptos_genesis::test_utils::test_config();
     config.storage.dir = checkpoint_dir.as_ref().to_path_buf();
     config.storage.storage_pruner_config = pruner_config;
-    config.storage.rocksdb_configs.use_state_kv_db = use_state_kv_db;
+    config.storage.rocksdb_configs.split_ledger_db = split_ledger_db;
     config.storage.rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db;
 
     let (db, executor) = init_db_and_executor::<V>(&config);
@@ -349,7 +356,7 @@ pub fn add_accounts<V>(
     checkpoint_dir: impl AsRef<Path>,
     pruner_config: PrunerConfig,
     verify_sequence_numbers: bool,
-    use_state_kv_db: bool,
+    split_ledger_db: bool,
     use_sharded_state_merkle_db: bool,
     pipeline_config: PipelineConfig,
 ) where
@@ -359,6 +366,7 @@ pub fn add_accounts<V>(
     create_checkpoint(
         source_dir.as_ref(),
         checkpoint_dir.as_ref(),
+        split_ledger_db,
         use_sharded_state_merkle_db,
     );
     add_accounts_impl::<V>(
@@ -369,7 +377,7 @@ pub fn add_accounts<V>(
         checkpoint_dir,
         pruner_config,
         verify_sequence_numbers,
-        use_state_kv_db,
+        split_ledger_db,
         use_sharded_state_merkle_db,
         pipeline_config,
     );
@@ -383,7 +391,7 @@ fn add_accounts_impl<V>(
     output_dir: impl AsRef<Path>,
     pruner_config: PrunerConfig,
     verify_sequence_numbers: bool,
-    use_state_kv_db: bool,
+    split_ledger_db: bool,
     use_sharded_state_merkle_db: bool,
     pipeline_config: PipelineConfig,
 ) where
@@ -392,7 +400,7 @@ fn add_accounts_impl<V>(
     let (mut config, genesis_key) = aptos_genesis::test_utils::test_config();
     config.storage.dir = output_dir.as_ref().to_path_buf();
     config.storage.storage_pruner_config = pruner_config;
-    config.storage.rocksdb_configs.use_state_kv_db = use_state_kv_db;
+    config.storage.rocksdb_configs.split_ledger_db = split_ledger_db;
     config.storage.rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db;
     let (db, executor) = init_db_and_executor::<V>(&config);
 

execution/executor-benchmark/src/main.rs

@@ -124,7 +124,7 @@ struct Opt {
     pruner_opt: PrunerOpt,
 
     #[clap(long)]
-    use_state_kv_db: bool,
+    split_ledger_db: bool,
 
     #[clap(long)]
     use_sharded_state_merkle_db: bool,
@@ -229,7 +229,7 @@ where
                 data_dir,
                 opt.pruner_opt.pruner_config(),
                 opt.verify_sequence_numbers,
-                opt.use_state_kv_db,
+                opt.split_ledger_db,
                 opt.use_sharded_state_merkle_db,
                 opt.pipeline_opt.pipeline_config(),
             );
@@ -254,7 +254,7 @@ where
                 checkpoint_dir,
                 opt.verify_sequence_numbers,
                 opt.pruner_opt.pruner_config(),
-                opt.use_state_kv_db,
+                opt.split_ledger_db,
                 opt.use_sharded_state_merkle_db,
                 opt.pipeline_opt.pipeline_config(),
             );
@@ -273,7 +273,7 @@ where
                 checkpoint_dir,
                 opt.pruner_opt.pruner_config(),
                 opt.verify_sequence_numbers,
-                opt.use_state_kv_db,
+                opt.split_ledger_db,
                 opt.use_sharded_state_merkle_db,
                 opt.pipeline_opt.pipeline_config(),
             );

storage/aptosdb/src/db_debugger/checkpoint/mod.rs

@@ -21,7 +21,7 @@ impl Cmd {
         ensure!(!self.output_dir.exists(), "Output dir already exists.");
         fs::create_dir_all(&self.output_dir)?;
 
-        // TODO(grao): Support sharded state merkle db here.
-        AptosDB::create_checkpoint(self.db_dir, self.output_dir, false)
+        // TODO(grao): Support sharded state merkle db and split_ledger_db here.
+        AptosDB::create_checkpoint(self.db_dir, self.output_dir, false, false)
     }
 }

storage/aptosdb/src/db_debugger/examine/print_db_versions.rs

@@ -33,13 +33,13 @@ pub struct Cmd {
     db_dir: PathBuf,
 
     #[clap(long)]
-    use_state_kv_db: bool,
+    split_ledger_db: bool,
 }
 
 impl Cmd {
     pub fn run(self) -> Result<()> {
         let rocksdb_config = RocksdbConfigs {
-            use_state_kv_db: self.use_state_kv_db,
+            split_ledger_db: self.split_ledger_db,
             ..Default::default()
         };
         let (ledger_db, state_merkle_db, state_kv_db) = AptosDB::open_dbs(

storage/aptosdb/src/db_debugger/truncate/mod.rs

@@ -47,7 +47,7 @@ pub struct Cmd {
     opt_out_backup_checkpoint: bool,
 
     #[clap(long)]
-    use_state_kv_db: bool,
+    split_ledger_db: bool,
 }
 
 impl Cmd {
@@ -61,14 +61,19 @@ impl Cmd {
             println!("Creating backup at: {:?}", &backup_checkpoint_dir);
             fs::create_dir_all(&backup_checkpoint_dir)?;
             // TODO(grao): Support sharded state merkle db here.
-            AptosDB::create_checkpoint(&self.db_dir, backup_checkpoint_dir, false)?;
+            AptosDB::create_checkpoint(
+                &self.db_dir,
+                backup_checkpoint_dir,
+                self.split_ledger_db,
+                false,
+            )?;
             println!("Done!");
         } else {
             println!("Opted out backup creation!.");
         }
 
         let rocksdb_config = RocksdbConfigs {
-            use_state_kv_db: self.use_state_kv_db,
+            split_ledger_db: self.split_ledger_db,
             ..Default::default()
         };
         let (ledger_db, state_merkle_db, state_kv_db) = AptosDB::open_dbs(
@@ -242,7 +247,7 @@ mod test {
                 ledger_db_batch_size: 15,
                 opt_out_backup_checkpoint: true,
                 backup_checkpoint_dir: None,
-                use_state_kv_db: false,
+                split_ledger_db: false,
             };
 
             cmd.run().unwrap();

storage/aptosdb/src/ledger_db.rs

@@ -133,11 +133,53 @@ impl LedgerDb {
     }
 
     pub(crate) fn create_checkpoint(
-        _db_root_path: impl AsRef<Path>,
-        _cp_root_path: impl AsRef<Path>,
+        db_root_path: impl AsRef<Path>,
+        cp_root_path: impl AsRef<Path>,
+        split_ledger_db: bool,
     ) -> Result<()> {
-        // TODO(grao): Implement this function.
-        todo!()
+        let rocksdb_configs = RocksdbConfigs {
+            split_ledger_db,
+            ..Default::default()
+        };
+        let ledger_db = Self::new(db_root_path, rocksdb_configs, /*readonly=*/ false)?;
+        let cp_ledger_db_folder = cp_root_path.as_ref().join(LEDGER_DB_FOLDER_NAME);
+
+        info!(
+            split_ledger_db = split_ledger_db,
+            "Creating ledger_db checkpoint at: {cp_ledger_db_folder:?}"
+        );
+
+        std::fs::remove_dir_all(&cp_ledger_db_folder).unwrap_or(());
+        if split_ledger_db {
+            std::fs::create_dir_all(&cp_ledger_db_folder).unwrap_or(());
+        }
+
+        ledger_db
+            .metadata_db()
+            .create_checkpoint(Self::metadata_db_path(
+                cp_root_path.as_ref(),
+                split_ledger_db,
+            ))?;
+
+        if split_ledger_db {
+            ledger_db
+                .event_db()
+                .create_checkpoint(cp_ledger_db_folder.join(EVENT_DB_NAME))?;
+            ledger_db
+                .transaction_accumulator_db()
+                .create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_ACCUMULATOR_DB_NAME))?;
+            ledger_db
+                .transaction_db()
+                .create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_DB_NAME))?;
+            ledger_db
+                .transaction_info_db()
+                .create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_INFO_DB_NAME))?;
+            ledger_db
+                .write_set_db()
+                .create_checkpoint(cp_ledger_db_folder.join(WRITE_SET_DB_NAME))?;
+        }
+
+        Ok(())
     }
 
     pub(crate) fn write_pruner_progress(&self, version: Version) -> Result<()> {

storage/aptosdb/src/lib.rs

@@ -703,27 +703,21 @@ impl AptosDB {
     pub fn create_checkpoint(
         db_path: impl AsRef<Path>,
         cp_path: impl AsRef<Path>,
+        use_split_ledger_db: bool,
         use_sharded_state_merkle_db: bool,
     ) -> Result<()> {
         let start = Instant::now();
-        let ledger_db_path = db_path.as_ref().join(LEDGER_DB_NAME);
-        let ledger_cp_path = cp_path.as_ref().join(LEDGER_DB_NAME);
 
-        info!("Creating ledger_db checkpoint at: {ledger_cp_path:?}");
-
-        std::fs::remove_dir_all(&ledger_cp_path).unwrap_or(());
-
-        // Weird enough, checkpoint doesn't work with readonly or secondary mode (gets stuck).
-        // https://github.com/facebook/rocksdb/issues/11167
-        let ledger_db = aptos_schemadb::DB::open(
-            ledger_db_path,
-            LEDGER_DB_NAME,
-            ledger_db_column_families(),
-            &aptos_schemadb::Options::default(),
-        )?;
-        ledger_db.create_checkpoint(ledger_cp_path)?;
+        info!(
+            use_split_ledger_db = use_split_ledger_db,
+            use_sharded_state_merkle_db = use_sharded_state_merkle_db,
+            "Creating checkpoint for AptosDB."
+        );
 
-        StateKvDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref())?;
+        LedgerDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref(), use_split_ledger_db)?;
+        if use_split_ledger_db {
+            StateKvDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref())?;
+        }
         StateMerkleDb::create_checkpoint(
             db_path.as_ref(),
             cp_path.as_ref(),

storage/aptosdb/src/state_kv_db.rs

@@ -38,7 +38,7 @@ impl StateKvDb {
         readonly: bool,
         ledger_db: Arc<DB>,
     ) -> Result<Self> {
-        if !rocksdb_configs.use_state_kv_db {
+        if !rocksdb_configs.split_ledger_db {
             info!("State K/V DB is not enabled!");
             return Ok(Self {
                 state_kv_metadata_db: Arc::clone(&ledger_db),

storage/backup/backup-cli/src/utils/mod.rs

@@ -69,8 +69,6 @@ pub struct RocksdbOpt {
     #[clap(long, hidden(true))]
     split_ledger_db: bool,
     #[clap(long, hidden(true))]
-    use_state_kv_db: bool,
-    #[clap(long, hidden(true))]
     use_sharded_state_merkle_db: bool,
     #[clap(long, hidden(true), default_value = "5000")]
     state_kv_db_max_open_files: i32,
@@ -100,7 +98,6 @@ impl From<RocksdbOpt> for RocksdbConfigs {
                 ..Default::default()
             },
             split_ledger_db: opt.split_ledger_db,
-            use_state_kv_db: opt.use_state_kv_db,
             use_sharded_state_merkle_db: opt.use_sharded_state_merkle_db,
             state_kv_db_config: RocksdbConfig {
                 max_open_files: opt.state_kv_db_max_open_files,