Skip to content

Commit

Permalink
[Storage][Sharding] Implement create_checkpoint for LedgerDb, and dep…
Browse files Browse the repository at this point in the history
…recate use_state_kv_db flag.
  • Loading branch information
grao1991 committed Jun 12, 2023
1 parent dd1cfd3 commit 54203e4
Show file tree
Hide file tree
Showing 15 changed files with 106 additions and 60 deletions.
1 change: 1 addition & 0 deletions aptos-node/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ fn create_rocksdb_checkpoint_and_change_working_dir(
AptosDB::create_checkpoint(
&source_dir,
&checkpoint_dir,
node_config.storage.rocksdb_configs.split_ledger_db,
node_config
.storage
.rocksdb_configs
Expand Down
4 changes: 0 additions & 4 deletions config/src/config/storage_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,6 @@ pub struct RocksdbConfigs {
pub ledger_db_config: RocksdbConfig,
pub state_merkle_db_config: RocksdbConfig,
// Note: Not ready for production use yet.
// TODO(grao): Deprecate this flag and use the split_ledger_db_to_individual_dbs below.
pub use_state_kv_db: bool,
// Note: Not ready for production use yet.
pub use_sharded_state_merkle_db: bool,
// Note: Not ready for production use yet.
// TODO(grao): Add RocksdbConfig for individual DBs when necessary.
Expand All @@ -81,7 +78,6 @@ impl Default for RocksdbConfigs {
Self {
ledger_db_config: RocksdbConfig::default(),
state_merkle_db_config: RocksdbConfig::default(),
use_state_kv_db: false,
use_sharded_state_merkle_db: false,
split_ledger_db: false,
state_kv_db_config: RocksdbConfig::default(),
Expand Down
15 changes: 10 additions & 5 deletions execution/executor-benchmark/src/db_generator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ pub fn create_db_with_accounts<V>(
db_dir: impl AsRef<Path>,
storage_pruner_config: PrunerConfig,
verify_sequence_numbers: bool,
use_state_kv_db: bool,
split_ledger_db: bool,
use_sharded_state_merkle_db: bool,
pipeline_config: PipelineConfig,
) where
Expand All @@ -40,7 +40,7 @@ pub fn create_db_with_accounts<V>(
// create if not exists
fs::create_dir_all(db_dir.as_ref()).unwrap();

bootstrap_with_genesis(&db_dir, use_state_kv_db);
bootstrap_with_genesis(&db_dir, split_ledger_db, use_sharded_state_merkle_db);

println!(
"Finished empty DB creation, DB dir: {}. Creating accounts now...",
Expand All @@ -55,18 +55,23 @@ pub fn create_db_with_accounts<V>(
&db_dir,
storage_pruner_config,
verify_sequence_numbers,
use_state_kv_db,
split_ledger_db,
use_sharded_state_merkle_db,
pipeline_config,
);
}

fn bootstrap_with_genesis(db_dir: impl AsRef<Path>, use_state_kv_db: bool) {
fn bootstrap_with_genesis(
db_dir: impl AsRef<Path>,
split_ledger_db: bool,
use_sharded_state_merkle_db: bool,
) {
let (config, _genesis_key) = aptos_genesis::test_utils::test_config();

let mut rocksdb_configs = RocksdbConfigs::default();
rocksdb_configs.state_merkle_db_config.max_open_files = -1;
rocksdb_configs.use_state_kv_db = use_state_kv_db;
rocksdb_configs.split_ledger_db = split_ledger_db;
rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db;
let (_db, db_rw) = DbReaderWriter::wrap(
AptosDB::open(
&db_dir,
Expand Down
24 changes: 16 additions & 8 deletions execution/executor-benchmark/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ where
fn create_checkpoint(
source_dir: impl AsRef<Path>,
checkpoint_dir: impl AsRef<Path>,
split_ledger_db: bool,
use_sharded_state_merkle_db: bool,
) {
// Create rocksdb checkpoint.
Expand All @@ -81,8 +82,13 @@ fn create_checkpoint(
}
std::fs::create_dir_all(checkpoint_dir.as_ref()).unwrap();

AptosDB::create_checkpoint(source_dir, checkpoint_dir, use_sharded_state_merkle_db)
.expect("db checkpoint creation fails.");
AptosDB::create_checkpoint(
source_dir,
checkpoint_dir,
split_ledger_db,
use_sharded_state_merkle_db,
)
.expect("db checkpoint creation fails.");
}

/// Runs the benchmark with given parameters.
Expand All @@ -98,7 +104,7 @@ pub fn run_benchmark<V>(
checkpoint_dir: impl AsRef<Path>,
verify_sequence_numbers: bool,
pruner_config: PrunerConfig,
use_state_kv_db: bool,
split_ledger_db: bool,
use_sharded_state_merkle_db: bool,
pipeline_config: PipelineConfig,
) where
Expand All @@ -107,13 +113,14 @@ pub fn run_benchmark<V>(
create_checkpoint(
source_dir.as_ref(),
checkpoint_dir.as_ref(),
split_ledger_db,
use_sharded_state_merkle_db,
);

let (mut config, genesis_key) = aptos_genesis::test_utils::test_config();
config.storage.dir = checkpoint_dir.as_ref().to_path_buf();
config.storage.storage_pruner_config = pruner_config;
config.storage.rocksdb_configs.use_state_kv_db = use_state_kv_db;
config.storage.rocksdb_configs.split_ledger_db = split_ledger_db;
config.storage.rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db;

let (db, executor) = init_db_and_executor::<V>(&config);
Expand Down Expand Up @@ -349,7 +356,7 @@ pub fn add_accounts<V>(
checkpoint_dir: impl AsRef<Path>,
pruner_config: PrunerConfig,
verify_sequence_numbers: bool,
use_state_kv_db: bool,
split_ledger_db: bool,
use_sharded_state_merkle_db: bool,
pipeline_config: PipelineConfig,
) where
Expand All @@ -359,6 +366,7 @@ pub fn add_accounts<V>(
create_checkpoint(
source_dir.as_ref(),
checkpoint_dir.as_ref(),
split_ledger_db,
use_sharded_state_merkle_db,
);
add_accounts_impl::<V>(
Expand All @@ -369,7 +377,7 @@ pub fn add_accounts<V>(
checkpoint_dir,
pruner_config,
verify_sequence_numbers,
use_state_kv_db,
split_ledger_db,
use_sharded_state_merkle_db,
pipeline_config,
);
Expand All @@ -383,7 +391,7 @@ fn add_accounts_impl<V>(
output_dir: impl AsRef<Path>,
pruner_config: PrunerConfig,
verify_sequence_numbers: bool,
use_state_kv_db: bool,
split_ledger_db: bool,
use_sharded_state_merkle_db: bool,
pipeline_config: PipelineConfig,
) where
Expand All @@ -392,7 +400,7 @@ fn add_accounts_impl<V>(
let (mut config, genesis_key) = aptos_genesis::test_utils::test_config();
config.storage.dir = output_dir.as_ref().to_path_buf();
config.storage.storage_pruner_config = pruner_config;
config.storage.rocksdb_configs.use_state_kv_db = use_state_kv_db;
config.storage.rocksdb_configs.split_ledger_db = split_ledger_db;
config.storage.rocksdb_configs.use_sharded_state_merkle_db = use_sharded_state_merkle_db;
let (db, executor) = init_db_and_executor::<V>(&config);

Expand Down
8 changes: 4 additions & 4 deletions execution/executor-benchmark/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ struct Opt {
pruner_opt: PrunerOpt,

#[clap(long)]
use_state_kv_db: bool,
split_ledger_db: bool,

#[clap(long)]
use_sharded_state_merkle_db: bool,
Expand Down Expand Up @@ -229,7 +229,7 @@ where
data_dir,
opt.pruner_opt.pruner_config(),
opt.verify_sequence_numbers,
opt.use_state_kv_db,
opt.split_ledger_db,
opt.use_sharded_state_merkle_db,
opt.pipeline_opt.pipeline_config(),
);
Expand All @@ -254,7 +254,7 @@ where
checkpoint_dir,
opt.verify_sequence_numbers,
opt.pruner_opt.pruner_config(),
opt.use_state_kv_db,
opt.split_ledger_db,
opt.use_sharded_state_merkle_db,
opt.pipeline_opt.pipeline_config(),
);
Expand All @@ -273,7 +273,7 @@ where
checkpoint_dir,
opt.pruner_opt.pruner_config(),
opt.verify_sequence_numbers,
opt.use_state_kv_db,
opt.split_ledger_db,
opt.use_sharded_state_merkle_db,
opt.pipeline_opt.pipeline_config(),
);
Expand Down
4 changes: 2 additions & 2 deletions storage/aptosdb/src/db_debugger/checkpoint/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ impl Cmd {
ensure!(!self.output_dir.exists(), "Output dir already exists.");
fs::create_dir_all(&self.output_dir)?;

// TODO(grao): Support sharded state merkle db here.
AptosDB::create_checkpoint(self.db_dir, self.output_dir, false)
// TODO(grao): Support sharded state merkle db and split_ledger_db here.
AptosDB::create_checkpoint(self.db_dir, self.output_dir, false, false)
}
}
4 changes: 2 additions & 2 deletions storage/aptosdb/src/db_debugger/examine/print_db_versions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,13 @@ pub struct Cmd {
db_dir: PathBuf,

#[clap(long)]
use_state_kv_db: bool,
split_ledger_db: bool,
}

impl Cmd {
pub fn run(self) -> Result<()> {
let rocksdb_config = RocksdbConfigs {
use_state_kv_db: self.use_state_kv_db,
split_ledger_db: self.split_ledger_db,
..Default::default()
};
let (ledger_db, state_merkle_db, state_kv_db) = AptosDB::open_dbs(
Expand Down
13 changes: 9 additions & 4 deletions storage/aptosdb/src/db_debugger/truncate/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ pub struct Cmd {
opt_out_backup_checkpoint: bool,

#[clap(long)]
use_state_kv_db: bool,
split_ledger_db: bool,
}

impl Cmd {
Expand All @@ -61,14 +61,19 @@ impl Cmd {
println!("Creating backup at: {:?}", &backup_checkpoint_dir);
fs::create_dir_all(&backup_checkpoint_dir)?;
// TODO(grao): Support sharded state merkle db here.
AptosDB::create_checkpoint(&self.db_dir, backup_checkpoint_dir, false)?;
AptosDB::create_checkpoint(
&self.db_dir,
backup_checkpoint_dir,
self.split_ledger_db,
false,
)?;
println!("Done!");
} else {
println!("Opted out backup creation!.");
}

let rocksdb_config = RocksdbConfigs {
use_state_kv_db: self.use_state_kv_db,
split_ledger_db: self.split_ledger_db,
..Default::default()
};
let (ledger_db, state_merkle_db, state_kv_db) = AptosDB::open_dbs(
Expand Down Expand Up @@ -242,7 +247,7 @@ mod test {
ledger_db_batch_size: 15,
opt_out_backup_checkpoint: true,
backup_checkpoint_dir: None,
use_state_kv_db: false,
split_ledger_db: false,
};

cmd.run().unwrap();
Expand Down
48 changes: 44 additions & 4 deletions storage/aptosdb/src/ledger_db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,11 +133,51 @@ impl LedgerDb {
}

pub(crate) fn create_checkpoint(
_db_root_path: impl AsRef<Path>,
_cp_root_path: impl AsRef<Path>,
db_root_path: impl AsRef<Path>,
cp_root_path: impl AsRef<Path>,
split_ledger_db: bool,
) -> Result<()> {
// TODO(grao): Implement this function.
todo!()
let rocksdb_configs = RocksdbConfigs {
split_ledger_db,
..Default::default()
};
let ledger_db = Self::new(db_root_path, rocksdb_configs, /*readonly=*/ false)?;
let cp_ledger_db_folder = cp_root_path.as_ref().join(LEDGER_DB_FOLDER_NAME);

info!(
split_ledger_db = split_ledger_db,
"Creating ledger_db checkpoint at: {cp_ledger_db_folder:?}"
);

std::fs::remove_dir_all(&cp_ledger_db_folder).unwrap_or(());
std::fs::create_dir_all(&cp_ledger_db_folder).unwrap_or(());

ledger_db
.metadata_db()
.create_checkpoint(Self::metadata_db_path(
cp_root_path.as_ref(),
split_ledger_db,
))?;

if split_ledger_db {
ledger_db
.event_db()
.create_checkpoint(cp_ledger_db_folder.join(EVENT_DB_NAME))?;
ledger_db
.transaction_accumulator_db()
.create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_ACCUMULATOR_DB_NAME))?;
ledger_db
.transaction_db()
.create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_DB_NAME))?;
ledger_db
.transaction_info_db()
.create_checkpoint(cp_ledger_db_folder.join(TRANSACTION_INFO_DB_NAME))?;
ledger_db
.write_set_db()
.create_checkpoint(cp_ledger_db_folder.join(WRITE_SET_DB_NAME))?;
}

Ok(())
}

pub(crate) fn write_pruner_progress(&self, version: Version) -> Result<()> {
Expand Down
26 changes: 10 additions & 16 deletions storage/aptosdb/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -703,27 +703,21 @@ impl AptosDB {
pub fn create_checkpoint(
db_path: impl AsRef<Path>,
cp_path: impl AsRef<Path>,
use_split_ledger_db: bool,
use_sharded_state_merkle_db: bool,
) -> Result<()> {
let start = Instant::now();
let ledger_db_path = db_path.as_ref().join(LEDGER_DB_NAME);
let ledger_cp_path = cp_path.as_ref().join(LEDGER_DB_NAME);

info!("Creating ledger_db checkpoint at: {ledger_cp_path:?}");

std::fs::remove_dir_all(&ledger_cp_path).unwrap_or(());

// Weird enough, checkpoint doesn't work with readonly or secondary mode (gets stuck).
// https://github.com/facebook/rocksdb/issues/11167
let ledger_db = aptos_schemadb::DB::open(
ledger_db_path,
LEDGER_DB_NAME,
ledger_db_column_families(),
&aptos_schemadb::Options::default(),
)?;
ledger_db.create_checkpoint(ledger_cp_path)?;
info!(
use_split_ledger_db = use_split_ledger_db,
use_sharded_state_merkle_db = use_sharded_state_merkle_db,
"Creating checkpoint for AptosDB."
);

StateKvDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref())?;
LedgerDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref(), use_split_ledger_db)?;
if use_split_ledger_db {
StateKvDb::create_checkpoint(db_path.as_ref(), cp_path.as_ref())?;
}
StateMerkleDb::create_checkpoint(
db_path.as_ref(),
cp_path.as_ref(),
Expand Down
2 changes: 1 addition & 1 deletion storage/aptosdb/src/state_kv_db.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ impl StateKvDb {
readonly: bool,
ledger_db: Arc<DB>,
) -> Result<Self> {
if !rocksdb_configs.use_state_kv_db {
if !rocksdb_configs.split_ledger_db {
info!("State K/V DB is not enabled!");
return Ok(Self {
state_kv_metadata_db: Arc::clone(&ledger_db),
Expand Down
3 changes: 0 additions & 3 deletions storage/backup/backup-cli/src/utils/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,6 @@ pub struct RocksdbOpt {
#[clap(long, hidden(true))]
split_ledger_db: bool,
#[clap(long, hidden(true))]
use_state_kv_db: bool,
#[clap(long, hidden(true))]
use_sharded_state_merkle_db: bool,
#[clap(long, hidden(true), default_value = "5000")]
state_kv_db_max_open_files: i32,
Expand Down Expand Up @@ -100,7 +98,6 @@ impl From<RocksdbOpt> for RocksdbConfigs {
..Default::default()
},
split_ledger_db: opt.split_ledger_db,
use_state_kv_db: opt.use_state_kv_db,
use_sharded_state_merkle_db: opt.use_sharded_state_merkle_db,
state_kv_db_config: RocksdbConfig {
max_open_files: opt.state_kv_db_max_open_files,
Expand Down
Loading

0 comments on commit 54203e4

Please sign in to comment.