Add a scanning results reader

Cargo.lock

@@ -2192,6 +2192,17 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonrpc"
+version = "0.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34efde8d2422fb79ed56db1d3aea8fa5b583351d15a26770cdee2f88813dd702"
+dependencies = [
+ "base64 0.13.1",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "jsonrpc-core"
 version = "18.0.0"
@@ -5996,6 +6007,7 @@ dependencies = [
  "color-eyre",
  "hex",
  "itertools 0.12.0",
+ "jsonrpc",
  "regex",
  "reqwest",
  "serde_json",
@@ -6005,9 +6017,12 @@ dependencies = [
  "tokio",
  "tracing-error",
  "tracing-subscriber",
+ "zcash_client_backend",
+ "zcash_primitives",
  "zebra-chain",
  "zebra-node-services",
  "zebra-rpc",
+ "zebra-scan",
 ]
 
 [[package]]

zebra-utils/Cargo.toml

@@ -36,6 +36,11 @@ name = "block-template-to-proposal"
 path = "src/bin/block-template-to-proposal/main.rs"
 required-features = ["getblocktemplate-rpcs"]
 
+[[bin]]
+name = "scanning-results-reader"
+path = "src/bin/scanning-results-reader/main.rs"
+required-features = ["shielded-scan"]
+
 [features]
 default = []
 
@@ -61,6 +66,13 @@ getblocktemplate-rpcs = [
     "zebra-chain/getblocktemplate-rpcs",
 ]
 
+shielded-scan = [
+    "jsonrpc",
+    "zcash_primitives",
+    "zcash_client_backend",
+    "zebra-scan"
+]
+
 [dependencies]
 color-eyre = "0.6.2"
 # This is a transitive dependency via color-eyre.
@@ -76,6 +88,7 @@ thiserror = "1.0.48"
 
 zebra-node-services = { path = "../zebra-node-services", version = "1.0.0-beta.32" }
 zebra-chain = { path = "../zebra-chain", version = "1.0.0-beta.32" }
+zebra-scan = { path = "../zebra-scan", version = "0.1.0-alpha.1", optional = true }
 
 # These crates are needed for the block-template-to-proposal binary
 zebra-rpc = { path = "../zebra-rpc", version = "1.0.0-beta.32", optional = true }
@@ -90,3 +103,8 @@ reqwest = { version = "0.11.22", default-features = false, features = ["rustls-t
 
 # These crates are needed for the zebra-checkpoints and search-issue-refs binaries
 tokio = { version = "1.34.0", features = ["full"], optional = true }
+
+jsonrpc = { version = "0.16.0", optional = true }
+
+zcash_primitives = { version = "0.13.0-rc.1", optional = true }
+zcash_client_backend = {version = "0.10.0-rc.1", optional = true}

zebra-utils/README.md

@@ -1,17 +1,16 @@
 # Zebra Utilities
 
-This crate contains tools for zebra maintainers.
-
-## Programs
+Tools for maintaining and testing Zebra:
 
 - [zebra-checkpoints](#zebra-checkpoints)
 - [zebrad-hash-lookup](#zebrad-hash-lookup)
 - [zebrad-log-filter](#zebrad-log-filter)
 - [zcash-rpc-diff](#zcash-rpc-diff)
+- [scanning-results-reader](#scanning-results-reader)
 
 Binaries are easier to use if they are located in your system execution path.
 
-### zebra-checkpoints
+## zebra-checkpoints
 
 This command generates a list of zebra checkpoints, and writes them to standard output. Each checkpoint consists of a block height and hash.
 
@@ -93,7 +92,7 @@ Then use the commands above to regenerate the checkpoints.
 - Open a pull request with the updated Mainnet and Testnet lists at:
   https://github.com/ZcashFoundation/zebra/pulls
 
-### zebrad-hash-lookup
+## zebrad-hash-lookup
 
 Given a block hash the script will get additional information using `zcash-cli`.
 
@@ -108,7 +107,7 @@ $
 ```
 This program is commonly used as part of `zebrad-log-filter` where hashes will be captured from `zebrad` output.
 
-### zebrad-log-filter
+## zebrad-log-filter
 
 The program is designed to filter the output from the zebra terminal or log file. Each time a hash is seen the script will capture it and get the additional information using `zebrad-hash-lookup`.
 
@@ -127,7 +126,7 @@ next: 00000001436277884eef900772f0fcec9566becccebaab4713fd665b60fab309
 ...
 ```
 
-### zcash-rpc-diff
+## zcash-rpc-diff
 
 This program compares `zebrad` and `zcashd` RPC responses.
 
@@ -188,3 +187,52 @@ You can override the binaries the script calls using these environmental variabl
 - `$ZCASH_CLI`
 - `$DIFF`
 - `$JQ`
+
+## Scanning Results Reader
+
+A utility for displaying Zebra's scanning results.
+
+### How It Works
+
+1. Opens Zebra's scanning storage and reads the results containing scanning keys
+   and TXIDs.
+2. Fetches the transactions by their TXIDs from Zebra using the
+   `getrawtransaction` RPC.
+3. Decrypts the tx outputs using the corresponding scanning key.
+4. Prints the memos in the outputs.
+
+### How to Try It
+
+#### Scan the Block Chain with Zebra
+
+1. Add a viewing key to your Zebra config file. For example:
+
+   ``` toml
+   [shielded_scan.sapling_keys_to_scan]
+   "zxviews1q0duytgcqqqqpqre26wkl45gvwwwd706xw608hucmvfalr759ejwf7qshjf5r9aa7323zulvz6plhttp5mltqcgs9t039cx2d09mgq05ts63n8u35hyv6h9nc9ctqqtue2u7cer2mqegunuulq2luhq3ywjcz35yyljewa4mgkgjzyfwh6fr6jd0dzd44ghk0nxdv2hnv4j5nxfwv24rwdmgllhe0p8568sgqt9ckt02v2kxf5ahtql6s0ltjpkckw8gtymxtxuu9gcr0swvz" = 1
+   ```
+   This key is from [ZECpages](https://zecpages.com/boardinfo).
+
+2. Make sure Zebra runs on Mainnet and listens on the default RPC port by having
+   the following in the same config file:
+
+    ``` toml
+    [network]
+    network = 'Mainnet'
+
+    [rpc]
+    listen_addr = "127.0.0.1:8232"
+    ```
+
+3. Compile and run Zebra with `--features "shielded-scan"` and your config file.
+   Zebra will start scanning the block chain and inform you about its progress
+   each 10 000 blocks in the log.
+
+#### Run the Reader
+
+4. To print the memos in outputs decryptable by the provided scanning key, run
+   the reader while also running Zebra. For example:
+
+   ``` bash
+   cargo run --release --features shielded-scan --bin scanning-results-reader
+   ```

zebra-utils/src/bin/scanning-results-reader/main.rs

@@ -0,0 +1,130 @@
+//! Displays Zebra's scanning results:
+//!
+//! 1. Opens Zebra's scanning storage and reads the results containing scanning keys and TXIDs.
+//! 2. Fetches the transactions by their TXIDs from Zebra using the `getrawtransaction` RPC.
+//! 3. Decrypts the tx outputs using the corresponding scanning key.
+//! 4. Prints the memos in the outputs.
+
+use std::collections::HashMap;
+
+use jsonrpc::simple_http::SimpleHttpTransport;
+use jsonrpc::Client;
+use serde_json::value::RawValue;
+
+use zcash_client_backend::decrypt_transaction;
+use zcash_client_backend::keys::UnifiedFullViewingKey;
+use zcash_primitives::consensus::{BlockHeight, BranchId};
+use zcash_primitives::transaction::Transaction;
+use zcash_primitives::zip32::AccountId;
+
+use zebra_scan::scan::sapling_key_to_scan_block_keys;
+use zebra_scan::{storage::Storage, Config};
+
+/// Prints the memos of transactions from Zebra's scanning results storage.
+///
+/// Reads the results storage, iterates through all decrypted memos, and prints the them to standard
+/// output. Filters out some frequent and uninteresting memos typically associated with ZECPages.
+///
+/// Notes:
+///
+/// - `#[allow(clippy::print_stdout)]` is set to allow usage of `println!` for displaying the memos.
+/// - This function expects Zebra's RPC server to be available.
+///
+/// # Panics
+///
+/// When:
+///
+/// - The Sapling key from the storage is not valid.
+/// - There is no diversifiable full viewing key (dfvk) available.
+/// - The RPC response cannot be decoded from a hex string to bytes.
+/// - The transaction fetched via RPC cannot be deserialized from raw bytes.
+#[allow(clippy::print_stdout)]
+pub fn main() {
+    let network = zcash_primitives::consensus::Network::MainNetwork;
+    let storage = Storage::new(&Config::default(), zebra_network(&network), true);
+    // If the first memo is empty, it doesn't get printed. But we never print empty memos anyway.
+    let mut prev_memo = "".to_owned();
+
+    for (key, _) in storage.sapling_keys_last_heights().iter() {
+        let dfvk = sapling_key_to_scan_block_keys(key, zebra_network(&network))
+            .expect("Scanning key from the storage should be valid")
+            .0
+            .into_iter()
+            .next()
+            .expect("There should be exactly one dfvk");
+
+        let ufvk_with_acc_id = HashMap::from([(
+            AccountId::from(1),
+            UnifiedFullViewingKey::new(Some(dfvk), None).expect("`dfvk` should be `Some`"),
+        )]);
+
+        for (height, txids) in storage.sapling_results(key) {
+            let height = BlockHeight::from_u32(height.0);
+
+            for txid in txids.iter() {
+                let tx = Transaction::read(
+                    &hex::decode(&get_tx_via_rpc(hex::encode(txid.bytes_in_display_order())))
+                        .expect("RPC response should be decodable from hex string to bytes")[..],
+                    BranchId::for_height(&network, height),
+                )
+                .expect("TX fetched via RPC should be deserializable from raw bytes");
+
+                for output in decrypt_transaction(&network, height, &tx, &ufvk_with_acc_id) {
+                    let memo = memo_bytes_to_string(output.memo.as_array());
+
+                    if !memo.is_empty()
+                        // Filter out some uninteresting and repeating memos from ZECPages.
+                        && !memo.contains("LIKE:")
+                        && !memo.contains("VOTE:")
+                        && memo != prev_memo
+                    {
+                        println!("{memo}\n");
+                        prev_memo = memo;
+                    }
+                }
+            }
+        }
+    }
+}
+
+/// Trims trailing zeroes from a memo, and returns the memo as a string.
+fn memo_bytes_to_string(memo: &[u8; 512]) -> String {
+    match memo.iter().rposition(|&byte| byte != 0) {
+        Some(i) => std::str::from_utf8(&memo[..=i]).unwrap_or("").to_owned(),
+        None => "".to_owned(),
+    }
+}
+
+/// Uses the `getrawtransaction` RPC to retrieve a transaction by its TXID.
+fn get_tx_via_rpc(txid: String) -> String {
+    // Wrap the TXID with `"` so that [`RawValue::from_string`] eats it.
+    let txid = format!("\"{}\"", txid);
+    let transport = SimpleHttpTransport::builder()
+        .url("127.0.0.1:8232")
+        .expect("URL should be valid")
+        .build();
+    let client = Client::with_transport(transport);
+    let params = [RawValue::from_string(txid).expect("Provided TXID should be a valid JSON")];
+    let request = client.build_request("getrawtransaction", &params);
+    let response = client
+        .send_request(request)
+        .expect("Sending the `getrawtransaction` request should succeed");
+
+    response
+        .result()
+        .expect("Zebra's RPC response should contain a valid result")
+}
+
+/// Converts [`zcash_primitives::consensus::Network`] to [`zebra_chain::parameters::Network`].
+fn zebra_network(
+    network: &zcash_primitives::consensus::Network,
+) -> zebra_chain::parameters::Network {
+    match network {
+        zcash_primitives::consensus::Network::MainNetwork => {
+            zebra_chain::parameters::Network::Mainnet
+        }
+        zcash_primitives::consensus::Network::TestNetwork => {
+            zebra_chain::parameters::Network::Testnet
+        }
+    }
+}