diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index dc930fb48..69185a2f2 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -49,6 +49,7 @@ rayon = "1.3.0" flexi_logger = "0.14.7" typenum = "1.11.2" generic-array = "0.13.2" +tempdir = "0.3.7" [features] default = ["gpu", "measurements"] diff --git a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs index 5d43ef22b..0b69b568d 100644 --- a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs +++ b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs @@ -1,32 +1,33 @@ -use std::fs::{create_dir_all, OpenOptions}; +use std::fs::{create_dir_all, remove_dir_all, OpenOptions}; use std::path::{Path, PathBuf}; -use anyhow::{Context, Result}; +use anyhow::{ensure, Context, Result}; +use bincode::deserialize; use clap::{value_t, App, Arg, SubCommand}; -use generic_array::typenum::{U0, U8}; +use generic_array::typenum::Unsigned; use memmap::MmapOptions; -use merkletree::store::StoreConfig; +use merkletree::merkle::get_merkle_tree_len; +use merkletree::store::{ExternalReader, ReplicaConfig, Store, StoreConfig}; +use tempdir::TempDir; use filecoin_proofs::constants::*; use filecoin_proofs::types::*; +use filecoin_proofs::with_shape; use storage_proofs::cache_key::CacheKey; -use storage_proofs::merkle::LCTree; -use storage_proofs::merkle::{get_base_tree_count, split_config_and_replica}; +use storage_proofs::hasher::Hasher; +use storage_proofs::merkle::{create_lc_tree, get_base_tree_count, split_config_and_replica}; +use storage_proofs::merkle::{LCStore, LCTree, MerkleTreeTrait}; use storage_proofs::util::{default_rows_to_discard, NODE_SIZE}; -fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Result<()> { - let tree_count = match sector_size as u64 { - SECTOR_SIZE_2_KIB => get_base_tree_count::(), - SECTOR_SIZE_4_KIB => get_base_tree_count::(), - SECTOR_SIZE_16_KIB => get_base_tree_count::(), - SECTOR_SIZE_32_KIB => get_base_tree_count::(), - SECTOR_SIZE_512_MIB => get_base_tree_count::(), - SECTOR_SIZE_32_GIB => get_base_tree_count::(), - SECTOR_SIZE_64_GIB => get_base_tree_count::(), - _ => panic!("Unsupported sector size"), - }; +fn get_tree_r_info( + sector_size: usize, + cache: &PathBuf, + replica_path: &PathBuf, +) -> Result<(usize, usize, Vec, ReplicaConfig)> { + let tree_count = with_shape!(sector_size as u64, get_base_tree_count); + // Number of nodes per base tree - let nodes_count = sector_size / NODE_SIZE / tree_count; + let base_tree_leafs = sector_size / NODE_SIZE / tree_count; // If the cache dir doesn't exist, create it if !Path::new(&cache).exists() { @@ -37,22 +38,86 @@ fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu let tree_r_last_config = StoreConfig::new( &cache, CacheKey::CommRLastTree.to_string(), - default_rows_to_discard(nodes_count, OCT_ARITY), - ); - println!( - "Using nodes_count {}, rows_to_discard {}", - nodes_count, - default_rows_to_discard(nodes_count, OCT_ARITY) + default_rows_to_discard(base_tree_leafs, OCT_ARITY), ); // Split the config based on the number of nodes required let (configs, replica_config) = split_config_and_replica( tree_r_last_config, replica_path.clone(), - nodes_count, + base_tree_leafs, tree_count, )?; + Ok((tree_count, base_tree_leafs, configs, replica_config)) +} + +fn get_tree_r_last_root( + base_tree_leafs: usize, + sector_size: u64, + configs: &[StoreConfig], + replica_config: &ReplicaConfig, +) -> Result { + let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?; + let tree_r_last_root = if is_sector_shape_base(sector_size) { + ensure!(configs.len() == 1, "Invalid tree-shape specified"); + let store = LCStore::::new_from_disk_with_reader( + base_tree_len, + OCT_ARITY, + &configs[0], + ExternalReader::new_from_path(&replica_config.path)?, + )?; + + let tree_r_last = SectorShapeBase::from_data_store(store, base_tree_leafs)?; + tree_r_last.root() + } else if is_sector_shape_sub2(sector_size) { + let tree_r_last = SectorShapeSub2::from_store_configs_and_replica( + base_tree_leafs, + &configs, + &replica_config, + )?; + tree_r_last.root() + } else if is_sector_shape_sub8(sector_size) { + let tree_r_last = SectorShapeSub8::from_store_configs_and_replica( + base_tree_leafs, + &configs, + &replica_config, + )?; + tree_r_last.root() + } else if is_sector_shape_top2(sector_size) { + let tree_r_last = SectorShapeTop2::from_sub_tree_store_configs_and_replica( + base_tree_leafs, + &configs, + &replica_config, + )?; + tree_r_last.root() + } else { + panic!("Unsupported sector size"); + }; + + Ok(tree_r_last_root) +} + +fn get_persistent_aux(cache: &PathBuf) -> Result> { + let p_aux: PersistentAux = { + let p_aux_path = cache.join(CacheKey::PAux.to_string()); + let p_aux_bytes = std::fs::read(&p_aux_path) + .with_context(|| format!("could not read file p_aux={:?}", p_aux_path))?; + + deserialize(&p_aux_bytes) + }?; + + Ok(p_aux) +} + +fn build_tree_r_last( + sector_size: usize, + cache: &PathBuf, + replica_path: &PathBuf, +) -> Result<(::Domain, Vec)> { + let (tree_count, base_tree_leafs, configs, replica_config) = + get_tree_r_info(sector_size, &cache, &replica_path)?; + let f_data = OpenOptions::new() .read(true) .write(true) @@ -64,6 +129,7 @@ fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu .with_context(|| format!("could not mmap replica_path={:?}", replica_path))? }; + let mut base_tree_roots: Vec = Vec::with_capacity(tree_count); for i in 0..tree_count { let config = &configs[i]; let offset = replica_config.offsets[i]; @@ -71,20 +137,170 @@ fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu let slice = &input_mmap[offset..(offset + (sector_size / tree_count))]; let store_path = StoreConfig::data_path(&config.path, &config.id); println!( - "Building tree_r_last {}/{}, {} nodes at replica offset {}-{} and storing in {:?}", + "Building tree_r_last {}/{}, [nodes={}, rows_to_discard={}, offsets={}-{}] in {:?}", i + 1, tree_count, - nodes_count, + base_tree_leafs, + config.rows_to_discard, offset, (offset + (sector_size / tree_count)), &store_path ); - LCTree::::from_byte_slice_with_config( - slice, - config.clone(), + let tree = SectorShapeBase::from_byte_slice_with_config(slice, config.clone())?; + base_tree_roots.push(tree.root()); + } + + let tree_r_last = create_lc_tree::< + LCTree, + >( + get_merkle_tree_len(base_tree_leafs, Tree::Arity::to_usize())?, + &configs, + &replica_config, + )?; + + Ok((tree_r_last.root(), base_tree_roots)) +} + +fn run_rebuild( + sector_size: usize, + cache: PathBuf, + replica_path: PathBuf, +) -> Result<(DefaultTreeDomain, Vec)> { + with_shape!( + sector_size as u64, + build_tree_r_last, + sector_size, + &cache, + &replica_path + ) +} + +fn run_inspect(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Result<()> { + let (_tree_count, base_tree_leafs, configs, replica_config) = + get_tree_r_info(sector_size, &cache, &replica_path)?; + let tree_r_last_root = get_tree_r_last_root( + base_tree_leafs, + sector_size as u64, + &configs, + &replica_config, + )?; + let p_aux = get_persistent_aux(&cache)?; + + println!("CommRLast from p_aux: {:?}", p_aux.comm_r_last); + println!( + "CommRLast [cached tree_r_last root]: {:?}", + tree_r_last_root + ); + let status = if tree_r_last_root == p_aux.comm_r_last { + "MATCH" + } else { + "MISMATCH" + }; + println!("Cached inspection shows a {} of CommRLast", status); + + Ok(()) +} + +fn run_verify(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Result<()> { + let (tree_count, base_tree_leafs, configs, replica_config) = + get_tree_r_info(sector_size, &cache, &replica_path)?; + let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?; + + let match_str = |a, b| -> &str { + if a == b { + "MATCH" + } else { + "MISMATCH" + } + }; + + // First, read the roots from the cached trees on disk + let mut cached_base_tree_roots: Vec = Vec::with_capacity(tree_count); + for i in 0..tree_count { + let store = LCStore::new_from_disk_with_reader( + base_tree_len, + OCT_ARITY, + &configs[i], + ExternalReader::new_from_config(&replica_config, i)?, )?; + cached_base_tree_roots.push(store.last()?); + } + + // Retrieve the tree_r_last root from the cached trees on disk. + let tree_r_last_root = get_tree_r_last_root( + base_tree_leafs, + sector_size as u64, + &configs, + &replica_config, + )?; + + // Read comm_r_last from the persistent aux in the cache dir + let p_aux: PersistentAux = { + let p_aux_path = cache.join(CacheKey::PAux.to_string()); + let p_aux_bytes = std::fs::read(&p_aux_path) + .with_context(|| format!("could not read file p_aux={:?}", p_aux_path))?; + + deserialize(&p_aux_bytes) + }?; + + // Rebuild each of the tree_r_last base trees (in a new temp dir so as not to interfere + // with any existing ones on disk) and check if the roots match what's cached on disk + let tmp_dir = TempDir::new("tree-r-last-verify")?; + let tmp_path = tmp_dir.path(); + create_dir_all(&tmp_path)?; + + let (rebuilt_tree_r_last_root, rebuilt_base_tree_roots) = + run_rebuild(sector_size, tmp_path.to_path_buf(), replica_path)?; + + remove_dir_all(&tmp_path)?; + + let status = match_str(tree_r_last_root, p_aux.comm_r_last); + let rebuilt_status = match_str(rebuilt_tree_r_last_root, p_aux.comm_r_last); + + println!(); + for (i, (cached_root, rebuilt_root)) in cached_base_tree_roots + .iter() + .zip(rebuilt_base_tree_roots) + .enumerate() + { + println!( + "tree_r_last {}/{} inspection shows a {} of base tree root {:?}", + i + 1, + tree_count, + match_str(*cached_root, rebuilt_root), + rebuilt_root + ); + if *cached_root != rebuilt_root { + println!( + "Cached root {:?}, Rebuilt root {:?}", + cached_root, rebuilt_root + ); + } } + println!(); + println!( + "CommRLast from p_aux : {:?}", + p_aux.comm_r_last + ); + println!( + "CommRLast [cached tree_r_last root] : {:?}", + tree_r_last_root + ); + println!( + "CommRLast [rebuilt tree_r_last root]: {:?}", + rebuilt_tree_r_last_root + ); + println!(); + println!( + " Cached inspection shows a {} of CommRLast {:?}", + status, tree_r_last_root + ); + println!( + "Rebuilt inspection shows a {} of CommRLast {:?}", + rebuilt_status, rebuilt_tree_r_last_root + ); + Ok(()) } @@ -97,8 +313,58 @@ fn main() -> Result<()> { Arg::with_name("size") .required(true) .long("size") - .default_value("32") - .help("The data size in GiB") + .default_value("34359738368") + .help("The data size in bytes") + .takes_value(true), + ) + .arg( + Arg::with_name("replica") + .long("replica") + .help("The replica file") + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name("cache") + .long("cache") + .help("The cache directory for the output trees") + .required(true) + .takes_value(true), + ); + + let inspect_cmd = SubCommand::with_name("inspect") + .about("Inspect tree_r_last trees and match with p_aux in cache") + .arg( + Arg::with_name("size") + .required(true) + .long("size") + .default_value("34359738368") + .help("The data size in bytes") + .takes_value(true), + ) + .arg( + Arg::with_name("replica") + .long("replica") + .help("The replica file") + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name("cache") + .long("cache") + .help("The cache directory for the output trees") + .required(true) + .takes_value(true), + ); + + let verify_cmd = SubCommand::with_name("verify") + .about("Verify tree_r_last trees and check for cache mis-match") + .arg( + Arg::with_name("size") + .required(true) + .long("size") + .default_value("34359738368") + .help("The data size in bytes") .takes_value(true), ) .arg( @@ -119,6 +385,8 @@ fn main() -> Result<()> { let matches = App::new("update_tree_r_cache") .version("0.1") .subcommand(rebuild_cmd) + .subcommand(inspect_cmd) + .subcommand(verify_cmd) .get_matches(); match matches.subcommand() { @@ -127,7 +395,21 @@ fn main() -> Result<()> { let replica = value_t!(m, "replica", PathBuf)?; let size = value_t!(m, "size", usize) .expect("could not convert `size` CLI argument to `usize`"); - run_update(size, cache, replica)?; + run_rebuild(size, cache, replica)?; + } + ("inspect", Some(m)) => { + let cache = value_t!(m, "cache", PathBuf)?; + let replica = value_t!(m, "replica", PathBuf)?; + let size = value_t!(m, "size", usize) + .expect("could not convert `size` CLI argument to `usize`"); + run_inspect(size, cache, replica)?; + } + ("verify", Some(m)) => { + let cache = value_t!(m, "cache", PathBuf)?; + let replica = value_t!(m, "replica", PathBuf)?; + let size = value_t!(m, "size", usize) + .expect("could not convert `size` CLI argument to `usize`"); + run_verify(size, cache, replica)?; } _ => panic!("Unrecognized subcommand"), } diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs index ad95d61c3..3989e400b 100644 --- a/filecoin-proofs/src/constants.rs +++ b/filecoin-proofs/src/constants.rs @@ -128,16 +128,54 @@ pub type DefaultBinaryTree = storage_proofs::merkle::BinaryMerkleTree; pub type DefaultOctLCTree = storage_proofs::merkle::OctLCMerkleTree; -pub type SectorShape2KiB = LCTree; -pub type SectorShape4KiB = LCTree; -pub type SectorShape16KiB = LCTree; -pub type SectorShape32KiB = LCTree; -pub type SectorShape8MiB = LCTree; -pub type SectorShape16MiB = LCTree; -pub type SectorShape512MiB = LCTree; -pub type SectorShape1GiB = LCTree; -pub type SectorShape32GiB = LCTree; -pub type SectorShape64GiB = LCTree; +// Generic shapes +pub type SectorShapeBase = LCTree; +pub type SectorShapeSub2 = LCTree; +pub type SectorShapeSub8 = LCTree; +pub type SectorShapeTop2 = LCTree; + +// Specific size constants by shape +pub type SectorShape2KiB = SectorShapeBase; +pub type SectorShape8MiB = SectorShapeBase; +pub type SectorShape512MiB = SectorShapeBase; + +pub type SectorShape4KiB = SectorShapeSub2; +pub type SectorShape16MiB = SectorShapeSub2; +pub type SectorShape1GiB = SectorShapeSub2; + +pub type SectorShape16KiB = SectorShapeSub8; +pub type SectorShape32GiB = SectorShapeSub8; + +pub type SectorShape32KiB = SectorShapeTop2; +pub type SectorShape64GiB = SectorShapeTop2; + +pub fn is_sector_shape_base(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_2_KIB | SECTOR_SIZE_8_MIB | SECTOR_SIZE_512_MIB => true, + _ => false, + } +} + +pub fn is_sector_shape_sub2(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_1_GIB => true, + _ => false, + } +} + +pub fn is_sector_shape_sub8(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_GIB => true, + _ => false, + } +} + +pub fn is_sector_shape_top2(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_32_KIB | SECTOR_SIZE_64_GIB => true, + _ => false, + } +} pub use storage_proofs::merkle::{DiskTree, LCTree};