diff --git a/Cargo.lock b/Cargo.lock index 720fe491fea..a9091d725b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2192,6 +2192,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonrpc" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34efde8d2422fb79ed56db1d3aea8fa5b583351d15a26770cdee2f88813dd702" +dependencies = [ + "base64 0.13.1", + "serde", + "serde_json", +] + [[package]] name = "jsonrpc-core" version = "18.0.0" @@ -5996,6 +6007,7 @@ dependencies = [ "color-eyre", "hex", "itertools 0.12.0", + "jsonrpc", "regex", "reqwest", "serde_json", @@ -6005,9 +6017,12 @@ dependencies = [ "tokio", "tracing-error", "tracing-subscriber", + "zcash_client_backend", + "zcash_primitives", "zebra-chain", "zebra-node-services", "zebra-rpc", + "zebra-scan", ] [[package]] diff --git a/zebra-chain/src/block/height.rs b/zebra-chain/src/block/height.rs index 7449a510094..e13f03f0869 100644 --- a/zebra-chain/src/block/height.rs +++ b/zebra-chain/src/block/height.rs @@ -2,6 +2,7 @@ use std::ops::{Add, Sub}; use thiserror::Error; +use zcash_primitives::consensus::BlockHeight; use crate::{serialization::SerializationError, BoxError}; @@ -105,6 +106,12 @@ impl Height { } } +impl From for BlockHeight { + fn from(height: Height) -> Self { + BlockHeight::from_u32(height.0) + } +} + /// A difference between two [`Height`]s, possibly negative. /// /// This can represent the difference between any height values, diff --git a/zebra-chain/src/primitives/zcash_primitives.rs b/zebra-chain/src/primitives/zcash_primitives.rs index 62f8bacd254..a9fbd9f4b95 100644 --- a/zebra-chain/src/primitives/zcash_primitives.rs +++ b/zebra-chain/src/primitives/zcash_primitives.rs @@ -345,3 +345,12 @@ impl From for zcash_primitives::consensus::Network { } } } + +impl From for Network { + fn from(network: zcash_primitives::consensus::Network) -> Self { + match network { + zcash_primitives::consensus::Network::MainNetwork => Network::Mainnet, + zcash_primitives::consensus::Network::TestNetwork => Network::Testnet, + } + } +} diff --git a/zebra-utils/Cargo.toml b/zebra-utils/Cargo.toml index 45578ad98ec..3ba4a037c3a 100644 --- a/zebra-utils/Cargo.toml +++ b/zebra-utils/Cargo.toml @@ -36,6 +36,11 @@ name = "block-template-to-proposal" path = "src/bin/block-template-to-proposal/main.rs" required-features = ["getblocktemplate-rpcs"] +[[bin]] +name = "scanning-results-reader" +path = "src/bin/scanning-results-reader/main.rs" +required-features = ["shielded-scan"] + [features] default = [] @@ -61,6 +66,14 @@ getblocktemplate-rpcs = [ "zebra-chain/getblocktemplate-rpcs", ] +shielded-scan = [ + "itertools", + "jsonrpc", + "zcash_primitives", + "zcash_client_backend", + "zebra-scan" +] + [dependencies] color-eyre = "0.6.2" # This is a transitive dependency via color-eyre. @@ -76,6 +89,7 @@ thiserror = "1.0.48" zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.32" } zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.32" } +zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.1", optional = true } # These crates are needed for the block-template-to-proposal binary zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.32", optional = true } @@ -90,3 +104,8 @@ reqwest = { version = "0.11.22", default-features = false, features = ["rustls-t # These crates are needed for the zebra-checkpoints and search-issue-refs binaries tokio = { version = "1.34.0", features = ["full"], optional = true } + +jsonrpc = { version = "0.16.0", optional = true } + +zcash_primitives = { version = "0.13.0-rc.1", optional = true } +zcash_client_backend = {version = "0.10.0-rc.1", optional = true} diff --git a/zebra-utils/README.md b/zebra-utils/README.md index 73e89c25a48..b38d4c4346f 100644 --- a/zebra-utils/README.md +++ b/zebra-utils/README.md @@ -1,17 +1,16 @@ # Zebra Utilities -This crate contains tools for zebra maintainers. - -## Programs +Tools for maintaining and testing Zebra: - [zebra-checkpoints](#zebra-checkpoints) - [zebrad-hash-lookup](#zebrad-hash-lookup) - [zebrad-log-filter](#zebrad-log-filter) - [zcash-rpc-diff](#zcash-rpc-diff) +- [scanning-results-reader](#scanning-results-reader) Binaries are easier to use if they are located in your system execution path. -### zebra-checkpoints +## zebra-checkpoints This command generates a list of zebra checkpoints, and writes them to standard output. Each checkpoint consists of a block height and hash. @@ -93,7 +92,7 @@ Then use the commands above to regenerate the checkpoints. - Open a pull request with the updated Mainnet and Testnet lists at: https://github.com/ZcashFoundation/zebra/pulls -### zebrad-hash-lookup +## zebrad-hash-lookup Given a block hash the script will get additional information using `zcash-cli`. @@ -108,7 +107,7 @@ $ ``` This program is commonly used as part of `zebrad-log-filter` where hashes will be captured from `zebrad` output. -### zebrad-log-filter +## zebrad-log-filter The program is designed to filter the output from the zebra terminal or log file. Each time a hash is seen the script will capture it and get the additional information using `zebrad-hash-lookup`. @@ -127,7 +126,7 @@ next: 00000001436277884eef900772f0fcec9566becccebaab4713fd665b60fab309 ... ``` -### zcash-rpc-diff +## zcash-rpc-diff This program compares `zebrad` and `zcashd` RPC responses. @@ -188,3 +187,52 @@ You can override the binaries the script calls using these environmental variabl - `$ZCASH_CLI` - `$DIFF` - `$JQ` + +## Scanning Results Reader + +A utility for displaying Zebra's scanning results. + +### How It Works + +1. Opens Zebra's scanning storage and reads the results containing scanning keys + and TXIDs. +2. Fetches the transactions by their TXIDs from Zebra using the + `getrawtransaction` RPC. +3. Decrypts the tx outputs using the corresponding scanning key. +4. Prints the memos in the outputs. + +### How to Try It + +#### Scan the Block Chain with Zebra + +1. Add a viewing key to your Zebra config file. For example: + + ``` toml + [shielded_scan.sapling_keys_to_scan] + "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz" = 1 + ``` + This key is from [ZECpages](https://zecpages.com/boardinfo). + +2. Make sure Zebra runs on Mainnet and listens on the default RPC port by having + the following in the same config file: + + ``` toml + [network] + network = 'Mainnet' + + [rpc] + listen_addr = "127.0.0.1:8232" + ``` + +3. Compile and run Zebra with `--features "shielded-scan"` and your config file. + Zebra will start scanning the block chain and inform you about its progress + each 10 000 blocks in the log. + +#### Run the Reader + +4. To print the memos in outputs decryptable by the provided scanning key, run + the reader while also running Zebra. For example: + + ``` bash + cargo run --release --features shielded-scan --bin scanning-results-reader + ``` diff --git a/zebra-utils/src/bin/scanning-results-reader/main.rs b/zebra-utils/src/bin/scanning-results-reader/main.rs new file mode 100644 index 00000000000..676c779d47d --- /dev/null +++ b/zebra-utils/src/bin/scanning-results-reader/main.rs @@ -0,0 +1,118 @@ +//! Displays Zebra's scanning results: +//! +//! 1. Opens Zebra's scanning storage and reads the results containing scanning keys and TXIDs. +//! 2. Fetches the transactions by their TXIDs from Zebra using the `getrawtransaction` RPC. +//! 3. Decrypts the tx outputs using the corresponding scanning key. +//! 4. Prints the memos in the outputs. + +use std::collections::HashMap; + +use hex::ToHex; +use itertools::Itertools; +use jsonrpc::simple_http::SimpleHttpTransport; +use jsonrpc::Client; +use serde_json::value::RawValue; + +use zcash_client_backend::decrypt_transaction; +use zcash_client_backend::keys::UnifiedFullViewingKey; +use zcash_primitives::consensus::{BlockHeight, BranchId}; +use zcash_primitives::transaction::Transaction; +use zcash_primitives::zip32::AccountId; + +use zebra_scan::scan::sapling_key_to_scan_block_keys; +use zebra_scan::{storage::Storage, Config}; + +/// Prints the memos of transactions from Zebra's scanning results storage. +/// +/// Reads the results storage, iterates through all decrypted memos, and prints the them to standard +/// output. Filters out some frequent and uninteresting memos typically associated with ZECPages. +/// +/// Notes: +/// +/// - `#[allow(clippy::print_stdout)]` is set to allow usage of `println!` for displaying the memos. +/// - This function expects Zebra's RPC server to be available. +/// +/// # Panics +/// +/// When: +/// +/// - The Sapling key from the storage is not valid. +/// - There is no diversifiable full viewing key (dfvk) available. +/// - The RPC response cannot be decoded from a hex string to bytes. +/// - The transaction fetched via RPC cannot be deserialized from raw bytes. +#[allow(clippy::print_stdout)] +pub fn main() { + let network = zcash_primitives::consensus::Network::MainNetwork; + let storage = Storage::new(&Config::default(), network.into(), true); + // If the first memo is empty, it doesn't get printed. But we never print empty memos anyway. + let mut prev_memo = "".to_owned(); + + for (key, _) in storage.sapling_keys_last_heights().iter() { + let dfvk = sapling_key_to_scan_block_keys(key, network.into()) + .expect("Scanning key from the storage should be valid") + .0 + .into_iter() + .exactly_one() + .expect("There should be exactly one dfvk"); + + let ufvk_with_acc_id = HashMap::from([( + AccountId::from(1), + UnifiedFullViewingKey::new(Some(dfvk), None).expect("`dfvk` should be `Some`"), + )]); + + for (height, txids) in storage.sapling_results(key) { + let height = BlockHeight::from(height); + + for txid in txids.iter() { + let tx = Transaction::read( + &hex::decode(&get_tx_via_rpc(txid.encode_hex())) + .expect("RPC response should be decodable from hex string to bytes")[..], + BranchId::for_height(&network, height), + ) + .expect("TX fetched via RPC should be deserializable from raw bytes"); + + for output in decrypt_transaction(&network, height, &tx, &ufvk_with_acc_id) { + let memo = memo_bytes_to_string(output.memo.as_array()); + + if !memo.is_empty() + // Filter out some uninteresting and repeating memos from ZECPages. + && !memo.contains("LIKE:") + && !memo.contains("VOTE:") + && memo != prev_memo + { + println!("{memo}\n"); + prev_memo = memo; + } + } + } + } + } +} + +/// Trims trailing zeroes from a memo, and returns the memo as a [`String`]. +fn memo_bytes_to_string(memo: &[u8; 512]) -> String { + match memo.iter().rposition(|&byte| byte != 0) { + Some(i) => String::from_utf8_lossy(&memo[..=i]).into_owned(), + None => "".to_owned(), + } +} + +/// Uses the `getrawtransaction` RPC to retrieve a transaction by its TXID. +fn get_tx_via_rpc(txid: String) -> String { + // Wrap the TXID with `"` so that [`RawValue::from_string`] eats it. + let txid = format!("\"{}\"", txid); + let transport = SimpleHttpTransport::builder() + .url("127.0.0.1:8232") + .expect("URL should be valid") + .build(); + let client = Client::with_transport(transport); + let params = [RawValue::from_string(txid).expect("Provided TXID should be a valid JSON")]; + let request = client.build_request("getrawtransaction", ¶ms); + let response = client + .send_request(request) + .expect("Sending the `getrawtransaction` request should succeed"); + + response + .result() + .expect("Zebra's RPC response should contain a valid result") +}