From 8d2d1e143f232a70304a24529e2b5786949e8ca2 Mon Sep 17 00:00:00 2001 From: steviez Date: Fri, 14 Jun 2024 11:46:12 -0500 Subject: [PATCH 1/8] ledger-tool: Print only the bank-hash for --print-bank-hash The bank-hash command was previously used in some of the scripts to spin up a cluster. The change that deprecated the bank-hash command broke the script. So, this change: - Makes --print-bank-hash ONLY print the hash, no additional text to avoid any overhead around parsing - Update the remote-node.sh script to use verify --print-bank-hash --- ledger-tool/src/main.rs | 6 +----- net/remote/remote-node.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index fee913c82b593d..8a2a762ad8896e 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1817,11 +1817,7 @@ fn main() { working_bank.print_accounts_stats(); } if print_bank_hash { - println!( - "Bank hash for slot {}: {}", - working_bank.slot(), - working_bank.hash() - ); + println!("{}", working_bank.hash()); } if write_bank_file { bank_hash_details::write_bank_hash_details_file(&working_bank) diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 71378019730f05..23585ef30c95db 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -263,7 +263,7 @@ EOF agave-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version if [[ -n "$maybeWaitForSupermajority" ]]; then - bankHash=$(agave-ledger-tool -l config/bootstrap-validator bank-hash --halt-at-slot 0) + bankHash=$(agave-ledger-tool -l config/bootstrap-validator verify --halt-at-slot 0 --print-bank-hash) shredVersion="$(cat "$SOLANA_CONFIG_DIR"/shred-version)" extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash --expected-shred-version $shredVersion" echo "$bankHash" > config/bank-hash From 2af39395693713c91fbf77f6ba7efefd8d606386 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 14:03:05 -0500 Subject: [PATCH 2/8] Make --print-bank-hash support json via --output json --- ledger-tool/src/main.rs | 10 +++++++++- ledger-tool/src/output.rs | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 8a2a762ad8896e..8ebd6d4b33778e 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -8,6 +8,7 @@ use { ledger_utils::*, output::{ output_account, AccountsOutputConfig, AccountsOutputMode, AccountsOutputStreamer, + SlotBankHash, }, program::*, }, @@ -1791,9 +1792,12 @@ fn main() { process_options.slot_callback = slot_callback; + let output_format = + OutputFormat::from_matches(arg_matches, "output_format", false); let print_accounts_stats = arg_matches.is_present("print_accounts_stats"); let print_bank_hash = arg_matches.is_present("print_bank_hash"); let write_bank_file = arg_matches.is_present("write_bank_file"); + let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); info!("genesis hash: {}", genesis_config.hash()); @@ -1817,7 +1821,11 @@ fn main() { working_bank.print_accounts_stats(); } if print_bank_hash { - println!("{}", working_bank.hash()); + let slot_bank_hash = SlotBankHash { + slot: working_bank.slot(), + hash: working_bank.hash(), + }; + println!("{}", output_format.formatted_string(&slot_bank_hash)); } if write_bank_file { bank_hash_details::write_bank_hash_details_file(&working_bank) diff --git a/ledger-tool/src/output.rs b/ledger-tool/src/output.rs index 3de08cec989806..1284d653209547 100644 --- a/ledger-tool/src/output.rs +++ b/ledger-tool/src/output.rs @@ -101,6 +101,22 @@ impl Display for SlotBounds<'_> { } } +#[derive(Serialize, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct SlotBankHash { + pub slot: Slot, + pub hash: Hash, +} + +impl VerboseDisplay for SlotBankHash {} +impl QuietDisplay for SlotBankHash {} + +impl Display for SlotBankHash { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + writeln!(f, "Bank hash for slot {}: {}", self.slot, self.hash) + } +} + fn writeln_entry(f: &mut dyn fmt::Write, i: usize, entry: &CliEntry, prefix: &str) -> fmt::Result { writeln!( f, From 12faa6a89e8b641d1e5942f27c3734e8ca165553 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 14:35:57 -0500 Subject: [PATCH 3/8] Make cluster setup nodes install jq --- net/gce.sh | 1 + net/scripts/install-jq.sh | 10 ++++++++++ 2 files changed, 11 insertions(+) create mode 100644 net/scripts/install-jq.sh diff --git a/net/gce.sh b/net/gce.sh index 058dcdcf0e7590..2c48cb7132f4a8 100755 --- a/net/gce.sh +++ b/net/gce.sh @@ -805,6 +805,7 @@ $( install-certbot.sh \ install-earlyoom.sh \ install-iftop.sh \ + install-jq.sh \ install-libssl-compatability.sh \ install-rsync.sh \ install-perf.sh \ diff --git a/net/scripts/install-jq.sh b/net/scripts/install-jq.sh new file mode 100644 index 00000000000000..9d7c837c8dd9f1 --- /dev/null +++ b/net/scripts/install-jq.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# +# jq setup +# +set -ex + +[[ $(uname) = Linux ]] || exit 1 +[[ $USER = root ]] || exit 1 + +apt-get --assume-yes install jq From f0227ccf65244647512f18e15238b0b6a655cc31 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 14:42:36 -0500 Subject: [PATCH 4/8] Update script to use jq to parse bank-hash from ledger-tool command --- net/remote/remote-node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 23585ef30c95db..05e2a3cad1a35d 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -263,7 +263,7 @@ EOF agave-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version if [[ -n "$maybeWaitForSupermajority" ]]; then - bankHash=$(agave-ledger-tool -l config/bootstrap-validator verify --halt-at-slot 0 --print-bank-hash) + bankHash=$(agave-ledger-tool -l config/bootstrap-validator verify --halt-at-slot 0 --print-bank-hash | jq -r .hash) shredVersion="$(cat "$SOLANA_CONFIG_DIR"/shred-version)" extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash --expected-shred-version $shredVersion" echo "$bankHash" > config/bank-hash From a99cd89195062d2e2fdd0796fa486129e18dcc25 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 14:48:33 -0500 Subject: [PATCH 5/8] Make install-jq.sh executable --- net/scripts/install-jq.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 net/scripts/install-jq.sh diff --git a/net/scripts/install-jq.sh b/net/scripts/install-jq.sh old mode 100644 new mode 100755 From 089c5c519f86ea9e038b762047781a5307cc8820 Mon Sep 17 00:00:00 2001 From: steviez Date: Mon, 17 Jun 2024 14:56:56 -0500 Subject: [PATCH 6/8] Make hash a string so it serializes as a base58 string and not bytes --- ledger-tool/src/main.rs | 2 +- ledger-tool/src/output.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 8ebd6d4b33778e..7ae39975545ff2 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1823,7 +1823,7 @@ fn main() { if print_bank_hash { let slot_bank_hash = SlotBankHash { slot: working_bank.slot(), - hash: working_bank.hash(), + hash: working_bank.hash().to_string(), }; println!("{}", output_format.formatted_string(&slot_bank_hash)); } diff --git a/ledger-tool/src/output.rs b/ledger-tool/src/output.rs index 1284d653209547..7ee2e7b2f5372d 100644 --- a/ledger-tool/src/output.rs +++ b/ledger-tool/src/output.rs @@ -105,7 +105,7 @@ impl Display for SlotBounds<'_> { #[serde(rename_all = "camelCase")] pub struct SlotBankHash { pub slot: Slot, - pub hash: Hash, + pub hash: String, } impl VerboseDisplay for SlotBankHash {} From ca7a098e22d1e7cf2e7ce8647a59b3b97651f4a3 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 18 Jun 2024 11:50:02 -0500 Subject: [PATCH 7/8] Forgot to specify --output json --- net/remote/remote-node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 05e2a3cad1a35d..1b15e4707351b3 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -263,7 +263,7 @@ EOF agave-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version if [[ -n "$maybeWaitForSupermajority" ]]; then - bankHash=$(agave-ledger-tool -l config/bootstrap-validator verify --halt-at-slot 0 --print-bank-hash | jq -r .hash) + bankHash=$(agave-ledger-tool -l config/bootstrap-validator verify --halt-at-slot 0 --print-bank-hash --output json | jq -r .hash) shredVersion="$(cat "$SOLANA_CONFIG_DIR"/shred-version)" extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash --expected-shred-version $shredVersion" echo "$bankHash" > config/bank-hash From ddb78bb81758eb121b0e27304d6970e6b899c984 Mon Sep 17 00:00:00 2001 From: steviez Date: Tue, 18 Jun 2024 12:58:50 -0500 Subject: [PATCH 8/8] Use quotation marks for jq query --- net/remote/remote-node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 1b15e4707351b3..882f7891702cde 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -263,7 +263,7 @@ EOF agave-ledger-tool -l config/bootstrap-validator shred-version --max-genesis-archive-unpacked-size 1073741824 | tee config/shred-version if [[ -n "$maybeWaitForSupermajority" ]]; then - bankHash=$(agave-ledger-tool -l config/bootstrap-validator verify --halt-at-slot 0 --print-bank-hash --output json | jq -r .hash) + bankHash=$(agave-ledger-tool -l config/bootstrap-validator verify --halt-at-slot 0 --print-bank-hash --output json | jq -r ".hash") shredVersion="$(cat "$SOLANA_CONFIG_DIR"/shred-version)" extraNodeArgs="$extraNodeArgs --expected-bank-hash $bankHash --expected-shred-version $shredVersion" echo "$bankHash" > config/bank-hash