From 3eeb3437a119bfad9c759cc816529cfcc34a1c91 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 17:44:04 -0500 Subject: [PATCH 1/2] Move snapshot related args into a helper --- ledger-tool/src/args.rs | 32 ++++++++++++++++++++++++++ ledger-tool/src/main.rs | 51 +++++++---------------------------------- 2 files changed, 40 insertions(+), 43 deletions(-) diff --git a/ledger-tool/src/args.rs b/ledger-tool/src/args.rs index 4895e86bd03e5d..2101cf6b0340e7 100644 --- a/ledger-tool/src/args.rs +++ b/ledger-tool/src/args.rs @@ -112,6 +112,38 @@ pub fn accounts_db_args<'a, 'b>() -> Box<[Arg<'a, 'b>]> { .into_boxed_slice() } +/// Returns the arguments that configure snapshot loading +pub fn snapshot_args<'a, 'b>() -> Box<[Arg<'a, 'b>]> { + vec![ + Arg::with_name("no_snapshot") + .long("no-snapshot") + .takes_value(false) + .help("Do not start from a local snapshot if present"), + Arg::with_name("snapshots") + .long("snapshots") + .alias("snapshot-archive-path") + .alias("full-snapshot-archive-path") + .value_name("DIR") + .takes_value(true) + .global(true) + .help("Use DIR for snapshot location [default: --ledger value]"), + Arg::with_name("incremental_snapshot_archive_path") + .long("incremental-snapshot-archive-path") + .value_name("DIR") + .takes_value(true) + .global(true) + .help("Use DIR for separate incremental snapshot location"), + Arg::with_name(use_snapshot_archives_at_startup::cli::NAME) + .long(use_snapshot_archives_at_startup::cli::LONG_ARG) + .takes_value(true) + .possible_values(use_snapshot_archives_at_startup::cli::POSSIBLE_VALUES) + .default_value(use_snapshot_archives_at_startup::cli::default_value_for_ledger_tool()) + .help(use_snapshot_archives_at_startup::cli::HELP) + .long_help(use_snapshot_archives_at_startup::cli::LONG_HELP), + ] + .into_boxed_slice() +} + /// Parse a `ProcessOptions` from subcommand arguments. This function attempts /// to parse all flags related to `ProcessOptions`; however, subcommands that /// use this function may not support all flags. diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 5f6eac351f2351..7d0ac740f5b436 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -43,7 +43,6 @@ use { blockstore_processor::{ ProcessSlotCallback, TransactionStatusMessage, TransactionStatusSender, }, - use_snapshot_archives_at_startup, }, solana_measure::{measure, measure::Measure}, solana_runtime::{ @@ -563,11 +562,8 @@ fn main() { solana_logger::setup_with_default_filter(); let accounts_db_config_args = accounts_db_args(); + let snapshot_config_args = snapshot_args(); - let no_snapshot_arg = Arg::with_name("no_snapshot") - .long("no-snapshot") - .takes_value(false) - .help("Do not start from a local snapshot if present"); let accounts_db_test_hash_calculation_arg = Arg::with_name("accounts_db_test_hash_calculation") .long("accounts-db-test-hash-calculation") .help("Enable hash calculation test"); @@ -641,14 +637,6 @@ fn main() { .multiple(true) .takes_value(true) .help("Log when transactions are processed that reference the given key(s)."); - let use_snapshot_archives_at_startup = - Arg::with_name(use_snapshot_archives_at_startup::cli::NAME) - .long(use_snapshot_archives_at_startup::cli::LONG_ARG) - .takes_value(true) - .possible_values(use_snapshot_archives_at_startup::cli::POSSIBLE_VALUES) - .default_value(use_snapshot_archives_at_startup::cli::default_value_for_ledger_tool()) - .help(use_snapshot_archives_at_startup::cli::HELP) - .long_help(use_snapshot_archives_at_startup::cli::LONG_HELP); let geyser_plugin_args = Arg::with_name("geyser_plugin_config") .long("geyser-plugin-config") @@ -733,24 +721,6 @@ fn main() { run fine with a reduced file descriptor limit while others will not", ), ) - .arg( - Arg::with_name("snapshots") - .long("snapshots") - .alias("snapshot-archive-path") - .alias("full-snapshot-archive-path") - .value_name("DIR") - .takes_value(true) - .global(true) - .help("Use DIR for snapshot location [default: --ledger value]"), - ) - .arg( - Arg::with_name("incremental_snapshot_archive_path") - .long("incremental-snapshot-archive-path") - .value_name("DIR") - .takes_value(true) - .global(true) - .help("Use DIR for separate incremental snapshot location"), - ) .arg( Arg::with_name("block_verification_method") .long("block-verification-method") @@ -846,23 +816,23 @@ fn main() { SubCommand::with_name("shred-version") .about("Prints the ledger's shred hash") .args(&accounts_db_config_args) + .args(&snapshot_config_args) .arg(&hard_forks_arg) .arg(&max_genesis_archive_unpacked_size_arg) - .arg(&use_snapshot_archives_at_startup), ) .subcommand( SubCommand::with_name("bank-hash") .about("Prints the hash of the working bank after reading the ledger") .args(&accounts_db_config_args) + .args(&snapshot_config_args) .arg(&max_genesis_archive_unpacked_size_arg) .arg(&halt_at_slot_arg) - .arg(&use_snapshot_archives_at_startup), ) .subcommand( SubCommand::with_name("verify") .about("Verify the ledger") .args(&accounts_db_config_args) - .arg(&no_snapshot_arg) + .args(&snapshot_config_args) .arg(&halt_at_slot_arg) .arg(&limit_load_slot_count_from_snapshot_arg) .arg(&verify_index_arg) @@ -875,7 +845,6 @@ fn main() { .arg(&debug_key_arg) .arg(&geyser_plugin_args) .arg(&log_messages_bytes_limit_arg) - .arg(&use_snapshot_archives_at_startup) .arg( Arg::with_name("skip_poh_verify") .long("skip-poh-verify") @@ -999,11 +968,10 @@ fn main() { SubCommand::with_name("graph") .about("Create a Graphviz rendering of the ledger") .args(&accounts_db_config_args) - .arg(&no_snapshot_arg) + .args(&snapshot_config_args) .arg(&halt_at_slot_arg) .arg(&hard_forks_arg) .arg(&max_genesis_archive_unpacked_size_arg) - .arg(&use_snapshot_archives_at_startup) .arg( Arg::with_name("include_all_votes") .long("include-all-votes") @@ -1033,13 +1001,12 @@ fn main() { SubCommand::with_name("create-snapshot") .about("Create a new ledger snapshot") .args(&accounts_db_config_args) - .arg(&no_snapshot_arg) + .args(&snapshot_config_args) .arg(&hard_forks_arg) .arg(&max_genesis_archive_unpacked_size_arg) .arg(&snapshot_version_arg) .arg(&geyser_plugin_args) .arg(&log_messages_bytes_limit_arg) - .arg(&use_snapshot_archives_at_startup) .arg( Arg::with_name("snapshot_slot") .index(1) @@ -1240,13 +1207,12 @@ fn main() { SubCommand::with_name("accounts") .about("Print account stats and contents after processing the ledger") .args(&accounts_db_config_args) - .arg(&no_snapshot_arg) + .args(&snapshot_config_args) .arg(&halt_at_slot_arg) .arg(&hard_forks_arg) .arg(&geyser_plugin_args) .arg(&log_messages_bytes_limit_arg) .arg(&accounts_data_encoding_arg) - .arg(&use_snapshot_archives_at_startup) .arg(&max_genesis_archive_unpacked_size_arg) .arg( Arg::with_name("include_sysvars") @@ -1295,13 +1261,12 @@ fn main() { SubCommand::with_name("capitalization") .about("Print capitalization (aka, total supply) while checksumming it") .args(&accounts_db_config_args) - .arg(&no_snapshot_arg) + .args(&snapshot_config_args) .arg(&halt_at_slot_arg) .arg(&hard_forks_arg) .arg(&max_genesis_archive_unpacked_size_arg) .arg(&geyser_plugin_args) .arg(&log_messages_bytes_limit_arg) - .arg(&use_snapshot_archives_at_startup) .arg( Arg::with_name("warp_epoch") .required(false) From 48cf47e5e387519ffa55b56813f88accb45deb68 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 17:55:55 -0500 Subject: [PATCH 2/2] Move snapshot path parsing into load_and_process_ledger() --- ledger-tool/src/ledger_utils.rs | 17 ++++++++--------- ledger-tool/src/main.rs | 27 +++++++-------------------- ledger-tool/src/program.rs | 28 +++++----------------------- 3 files changed, 20 insertions(+), 52 deletions(-) diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 44b6667f0516f5..64e3ff64dbffb2 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -98,8 +98,6 @@ pub fn load_and_process_ledger_or_exit( genesis_config: &GenesisConfig, blockstore: Arc, process_options: ProcessOptions, - snapshot_archive_path: Option, - incremental_snapshot_archive_path: Option, transaction_status_sender: Option, ) -> (Arc>, Option) { load_and_process_ledger( @@ -107,8 +105,6 @@ pub fn load_and_process_ledger_or_exit( genesis_config, blockstore, process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, transaction_status_sender, ) .unwrap_or_else(|err| { @@ -122,8 +118,6 @@ pub fn load_and_process_ledger( genesis_config: &GenesisConfig, blockstore: Arc, process_options: ProcessOptions, - snapshot_archive_path: Option, - incremental_snapshot_archive_path: Option, transaction_status_sender: Option, ) -> Result<(Arc>, Option), LoadAndProcessLedgerError> { let bank_snapshots_dir = if blockstore.is_primary_access() { @@ -139,10 +133,15 @@ pub fn load_and_process_ledger( let snapshot_config = if arg_matches.is_present("no_snapshot") { None } else { - let full_snapshot_archives_dir = - snapshot_archive_path.unwrap_or_else(|| blockstore.ledger_path().to_path_buf()); + let full_snapshot_archives_dir = value_t!(arg_matches, "snapshots", String) + .ok() + .map(PathBuf::from) + .unwrap_or_else(|| blockstore.ledger_path().to_path_buf()); let incremental_snapshot_archives_dir = - incremental_snapshot_archive_path.unwrap_or_else(|| full_snapshot_archives_dir.clone()); + value_t!(arg_matches, "incremental_snapshot_archive_path", String) + .ok() + .map(PathBuf::from) + .unwrap_or_else(|| full_snapshot_archives_dir.clone()); if let Some(full_snapshot_slot) = snapshot_utils::get_highest_full_snapshot_archive_slot(&full_snapshot_archives_dir) { diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 7d0ac740f5b436..816acf546264e5 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1337,14 +1337,6 @@ fn main() { info!("{} {}", crate_name!(), solana_version::version!()); let ledger_path = PathBuf::from(value_t_or_exit!(matches, "ledger_path", String)); - let snapshot_archive_path = value_t!(matches, "snapshots", String) - .ok() - .map(PathBuf::from); - let incremental_snapshot_archive_path = - value_t!(matches, "incremental_snapshot_archive_path", String) - .ok() - .map(PathBuf::from); - let verbose_level = matches.occurrences_of("verbose"); // Name the rayon global thread pool @@ -1452,8 +1444,6 @@ fn main() { &genesis_config, Arc::new(blockstore), process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, None, ); @@ -1644,8 +1634,6 @@ fn main() { &genesis_config, Arc::new(blockstore), process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, transaction_status_sender, ); @@ -1712,8 +1700,6 @@ fn main() { &genesis_config, Arc::new(blockstore), process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, None, ); @@ -1738,6 +1724,13 @@ fn main() { let is_minimized = arg_matches.is_present("minimized"); let output_directory = value_t!(arg_matches, "output_directory", PathBuf) .unwrap_or_else(|_| { + let snapshot_archive_path = value_t!(matches, "snapshots", String) + .ok() + .map(PathBuf::from); + let incremental_snapshot_archive_path = + value_t!(matches, "incremental_snapshot_archive_path", String) + .ok() + .map(PathBuf::from); match ( is_incremental, &snapshot_archive_path, @@ -1876,8 +1869,6 @@ fn main() { &genesis_config, blockstore.clone(), process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, None, ); let mut bank = bank_forks @@ -2266,8 +2257,6 @@ fn main() { &genesis_config, Arc::new(blockstore), process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, None, ); let bank = bank_forks.read().unwrap().working_bank(); @@ -2319,8 +2308,6 @@ fn main() { &genesis_config, Arc::new(blockstore), process_options, - snapshot_archive_path, - incremental_snapshot_archive_path, None, ); let bank_forks = bank_forks.read().unwrap(); diff --git a/ledger-tool/src/program.rs b/ledger-tool/src/program.rs index 15750290fbed2a..9748b82a40d612 100644 --- a/ledger-tool/src/program.rs +++ b/ledger-tool/src/program.rs @@ -1,6 +1,6 @@ use { crate::{args::*, canonicalize_ledger_path, ledger_utils::*}, - clap::{value_t, App, AppSettings, Arg, ArgMatches, SubCommand}, + clap::{App, AppSettings, Arg, ArgMatches, SubCommand}, log::*, serde_derive::{Deserialize, Serialize}, serde_json::Result, @@ -9,7 +9,7 @@ use { syscalls::create_program_runtime_environment_v1, }, solana_cli_output::{OutputFormat, QuietDisplay, VerboseDisplay}, - solana_ledger::{blockstore_options::AccessType, use_snapshot_archives_at_startup}, + solana_ledger::blockstore_options::AccessType, solana_program_runtime::{ invoke_context::InvokeContext, loaded_programs::{ @@ -36,7 +36,7 @@ use { fmt::{self, Debug, Formatter}, fs::File, io::{Read, Seek, Write}, - path::{Path, PathBuf}, + path::Path, process::exit, sync::Arc, time::{Duration, Instant}, @@ -75,13 +75,6 @@ fn load_accounts(path: &Path) -> Result { fn load_blockstore(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> Arc { let process_options = parse_process_options(ledger_path, arg_matches); - let snapshot_archive_path = value_t!(arg_matches, "snapshots", String) - .ok() - .map(PathBuf::from); - let incremental_snapshot_archive_path = - value_t!(arg_matches, "incremental_snapshot_archive_path", String) - .ok() - .map(PathBuf::from); let genesis_config = open_genesis_config_by(ledger_path, arg_matches); info!("genesis hash: {}", genesis_config.hash()); @@ -91,8 +84,6 @@ fn load_blockstore(ledger_path: &Path, arg_matches: &ArgMatches<'_>) -> Arc { .takes_value(true) .default_value("10485760") .help("maximum total uncompressed size of unpacked genesis archive"); - let use_snapshot_archives_at_startup = - Arg::with_name(use_snapshot_archives_at_startup::cli::NAME) - .long(use_snapshot_archives_at_startup::cli::LONG_ARG) - .takes_value(true) - .possible_values(use_snapshot_archives_at_startup::cli::POSSIBLE_VALUES) - .default_value( - use_snapshot_archives_at_startup::cli::default_value_for_ledger_tool(), - ) - .help(use_snapshot_archives_at_startup::cli::HELP) - .long_help(use_snapshot_archives_at_startup::cli::LONG_HELP); + let snapshot_config_args = snapshot_args(); self.subcommand( SubCommand::with_name("program") @@ -179,8 +161,8 @@ and the following fields are required .takes_value(true) .default_value("0"), ) + .args(&snapshot_config_args) .arg(&max_genesis_arg) - .arg(&use_snapshot_archives_at_startup) .arg( Arg::with_name("memory") .help("Heap memory for the program to run on")