From ba152e6171df6c69369e02d709a4b5f652cc3aec Mon Sep 17 00:00:00 2001 From: nemo Date: Wed, 17 Jun 2020 16:57:35 -0400 Subject: [PATCH 1/3] feat: extend update_tree_r_cache command with new features Added are 'inspect' and 'verify' options. Inspect makes sure the cache is consistent with itself and verify rebuilds trees and then verifies against the cached data (which can detect an error in the replica data). --- fil-proofs-tooling/Cargo.toml | 1 + .../src/bin/update_tree_r_cache/main.rs | 355 ++++++++++++++++-- 2 files changed, 335 insertions(+), 21 deletions(-) diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index dc930fb48..69185a2f2 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -49,6 +49,7 @@ rayon = "1.3.0" flexi_logger = "0.14.7" typenum = "1.11.2" generic-array = "0.13.2" +tempdir = "0.3.7" [features] default = ["gpu", "measurements"] diff --git a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs index 5d43ef22b..1ba8a79fd 100644 --- a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs +++ b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs @@ -1,20 +1,28 @@ -use std::fs::{create_dir_all, OpenOptions}; +use std::fs::{create_dir_all, remove_dir_all, OpenOptions}; use std::path::{Path, PathBuf}; -use anyhow::{Context, Result}; +use anyhow::{ensure, Context, Result}; +use bincode::deserialize; use clap::{value_t, App, Arg, SubCommand}; -use generic_array::typenum::{U0, U8}; +use generic_array::typenum::{self, Unsigned}; use memmap::MmapOptions; -use merkletree::store::StoreConfig; +use merkletree::merkle::get_merkle_tree_len; +use merkletree::store::{ExternalReader, ReplicaConfig, Store, StoreConfig}; +use tempdir::TempDir; +use typenum::{U0, U8}; use filecoin_proofs::constants::*; use filecoin_proofs::types::*; use storage_proofs::cache_key::CacheKey; -use storage_proofs::merkle::LCTree; -use storage_proofs::merkle::{get_base_tree_count, split_config_and_replica}; +use storage_proofs::merkle::{create_lc_tree, get_base_tree_count, split_config_and_replica}; +use storage_proofs::merkle::{LCStore, LCTree, MerkleTreeTrait}; use storage_proofs::util::{default_rows_to_discard, NODE_SIZE}; -fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Result<()> { +fn get_tree_r_info( + sector_size: usize, + cache: &PathBuf, + replica_path: &PathBuf, +) -> Result<(usize, usize, Vec, ReplicaConfig)> { let tree_count = match sector_size as u64 { SECTOR_SIZE_2_KIB => get_base_tree_count::(), SECTOR_SIZE_4_KIB => get_base_tree_count::(), @@ -25,8 +33,9 @@ fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu SECTOR_SIZE_64_GIB => get_base_tree_count::(), _ => panic!("Unsupported sector size"), }; + // Number of nodes per base tree - let nodes_count = sector_size / NODE_SIZE / tree_count; + let base_tree_leafs = sector_size / NODE_SIZE / tree_count; // If the cache dir doesn't exist, create it if !Path::new(&cache).exists() { @@ -37,22 +46,81 @@ fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu let tree_r_last_config = StoreConfig::new( &cache, CacheKey::CommRLastTree.to_string(), - default_rows_to_discard(nodes_count, OCT_ARITY), - ); - println!( - "Using nodes_count {}, rows_to_discard {}", - nodes_count, - default_rows_to_discard(nodes_count, OCT_ARITY) + default_rows_to_discard(base_tree_leafs, OCT_ARITY), ); // Split the config based on the number of nodes required let (configs, replica_config) = split_config_and_replica( tree_r_last_config, replica_path.clone(), - nodes_count, + base_tree_leafs, tree_count, )?; + Ok((tree_count, base_tree_leafs, configs, replica_config)) +} + +fn get_tree_r_last_root( + base_tree_leafs: usize, + sector_size: usize, + configs: &[StoreConfig], + replica_config: &ReplicaConfig, +) -> Result { + let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?; + let tree_r_last_root = match sector_size as u64 { + SECTOR_SIZE_2_KIB | SECTOR_SIZE_8_MIB | SECTOR_SIZE_512_MIB => { + ensure!(configs.len() == 1, "Invalid tree-shape specified"); + let store = LCStore::::new_from_disk_with_reader( + base_tree_len, + OCT_ARITY, + &configs[0], + ExternalReader::new_from_path(&replica_config.path)?, + )?; + + let tree_r_last = LCTree::::from_data_store(store, base_tree_leafs)?; + tree_r_last.root() + } + SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_1_GIB => { + let tree_r_last = LCTree::::from_store_configs_and_replica(base_tree_leafs, &configs, &replica_config)?; + tree_r_last.root() + } + SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_GIB => { + let tree_r_last = LCTree::::from_store_configs_and_replica(base_tree_leafs, &configs, &replica_config)?; + tree_r_last.root() + } + SECTOR_SIZE_32_KIB | SECTOR_SIZE_64_GIB => { + let tree_r_last = LCTree::::from_sub_tree_store_configs_and_replica(base_tree_leafs, &configs, &replica_config)?; + tree_r_last.root() + } + _ => panic!("Unsupported sector size"), + }; + + Ok(tree_r_last_root) +} + +fn get_persistent_aux(cache: &PathBuf) -> Result> { + let p_aux: PersistentAux = { + let p_aux_path = cache.join(CacheKey::PAux.to_string()); + let p_aux_bytes = std::fs::read(&p_aux_path) + .with_context(|| format!("could not read file p_aux={:?}", p_aux_path))?; + + deserialize(&p_aux_bytes) + }?; + + Ok(p_aux) +} + +fn build_tree_r_last( + sector_size: usize, + cache: &PathBuf, + replica_path: &PathBuf, +) -> Result<( + LCTree, + Vec, +)> { + let (tree_count, base_tree_leafs, configs, replica_config) = + get_tree_r_info(sector_size, &cache, &replica_path)?; + let f_data = OpenOptions::new() .read(true) .write(true) @@ -64,6 +132,7 @@ fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu .with_context(|| format!("could not mmap replica_path={:?}", replica_path))? }; + let mut base_tree_roots: Vec = Vec::with_capacity(tree_count); for i in 0..tree_count { let config = &configs[i]; let offset = replica_config.offsets[i]; @@ -71,20 +140,198 @@ fn run_update(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu let slice = &input_mmap[offset..(offset + (sector_size / tree_count))]; let store_path = StoreConfig::data_path(&config.path, &config.id); println!( - "Building tree_r_last {}/{}, {} nodes at replica offset {}-{} and storing in {:?}", + "Building tree_r_last {}/{}, [nodes={}, rows_to_discard={}, offsets={}-{}] in {:?}", i + 1, tree_count, - nodes_count, + base_tree_leafs, + config.rows_to_discard, offset, (offset + (sector_size / tree_count)), &store_path ); - LCTree::::from_byte_slice_with_config( + let tree = LCTree::::from_byte_slice_with_config( slice, config.clone(), )?; + base_tree_roots.push(tree.root()); } + let tree_r_last = create_lc_tree::< + LCTree, + >( + get_merkle_tree_len(base_tree_leafs, Tree::Arity::to_usize())?, + &configs, + &replica_config, + )?; + + Ok((tree_r_last, base_tree_roots)) +} + +fn run_rebuild( + sector_size: usize, + cache: PathBuf, + replica_path: PathBuf, +) -> Result<(DefaultTreeDomain, Vec)> { + let (tree_r_last_root, base_tree_roots) = match sector_size as u64 { + SECTOR_SIZE_2_KIB => { + let (tree_r_last, base_tree_roots) = + build_tree_r_last::(sector_size, &cache, &replica_path)?; + (tree_r_last.root(), base_tree_roots) + } + SECTOR_SIZE_4_KIB => { + let (tree_r_last, base_tree_roots) = + build_tree_r_last::(sector_size, &cache, &replica_path)?; + (tree_r_last.root(), base_tree_roots) + } + SECTOR_SIZE_16_KIB => { + let (tree_r_last, base_tree_roots) = + build_tree_r_last::(sector_size, &cache, &replica_path)?; + (tree_r_last.root(), base_tree_roots) + } + SECTOR_SIZE_32_KIB => { + let (tree_r_last, base_tree_roots) = + build_tree_r_last::(sector_size, &cache, &replica_path)?; + (tree_r_last.root(), base_tree_roots) + } + SECTOR_SIZE_512_MIB => { + let (tree_r_last, base_tree_roots) = + build_tree_r_last::(sector_size, &cache, &replica_path)?; + (tree_r_last.root(), base_tree_roots) + } + SECTOR_SIZE_32_GIB => { + let (tree_r_last, base_tree_roots) = + build_tree_r_last::(sector_size, &cache, &replica_path)?; + (tree_r_last.root(), base_tree_roots) + } + SECTOR_SIZE_64_GIB => { + let (tree_r_last, base_tree_roots) = + build_tree_r_last::(sector_size, &cache, &replica_path)?; + (tree_r_last.root(), base_tree_roots) + } + _ => panic!("Unsupported sector size"), + }; + + Ok((tree_r_last_root, base_tree_roots)) +} + +fn run_inspect(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Result<()> { + let (_tree_count, base_tree_leafs, configs, replica_config) = + get_tree_r_info(sector_size, &cache, &replica_path)?; + let tree_r_last_root = + get_tree_r_last_root(base_tree_leafs, sector_size, &configs, &replica_config)?; + let p_aux = get_persistent_aux(&cache)?; + + println!("CommRLast from p_aux: {:?}", p_aux.comm_r_last); + println!( + "CommRLast [cached tree_r_last root]: {:?}", + tree_r_last_root + ); + let status = if tree_r_last_root == p_aux.comm_r_last { + "MATCH" + } else { + "MIS-MATCH" + }; + println!("Cached inspection shows a {} of CommRLast", status); + + Ok(()) +} + +fn run_verify(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Result<()> { + let (tree_count, base_tree_leafs, configs, replica_config) = + get_tree_r_info(sector_size, &cache, &replica_path)?; + let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?; + + let match_str = |a, b| -> &str { + if a == b { + "MATCH" + } else { + "MIS-MATCH" + } + }; + + // First, read the roots from the cached trees on disk + let mut cached_base_tree_roots: Vec = Vec::with_capacity(tree_count); + for i in 0..tree_count { + let store = LCStore::new_from_disk_with_reader( + base_tree_len, + OCT_ARITY, + &configs[i], + ExternalReader::new_from_config(&replica_config, i)?, + )?; + cached_base_tree_roots.push(store.last()?); + } + + // Retrieve the tree_r_last root from the cached trees on disk. + let tree_r_last_root = + get_tree_r_last_root(base_tree_leafs, sector_size, &configs, &replica_config)?; + + // Read comm_r_last from the persistent aux in the cache dir + let p_aux: PersistentAux = { + let p_aux_path = cache.join(CacheKey::PAux.to_string()); + let p_aux_bytes = std::fs::read(&p_aux_path) + .with_context(|| format!("could not read file p_aux={:?}", p_aux_path))?; + + deserialize(&p_aux_bytes) + }?; + + // Rebuild each of the tree_r_last base trees (in a new temp dir so as not to interfere + // with any existing ones on disk) and check if the roots match what's cached on disk + let tmp_dir = TempDir::new("tree-r-last-verify")?; + let tmp_path = tmp_dir.path(); + create_dir_all(&tmp_path)?; + + let (rebuilt_tree_r_last_root, rebuilt_base_tree_roots) = + run_rebuild(sector_size, tmp_path.to_path_buf(), replica_path)?; + + remove_dir_all(&tmp_path)?; + + let status = match_str(tree_r_last_root, p_aux.comm_r_last); + let rebuilt_status = match_str(rebuilt_tree_r_last_root, p_aux.comm_r_last); + + println!(); + for (i, (cached_root, rebuilt_root)) in cached_base_tree_roots + .iter() + .zip(rebuilt_base_tree_roots) + .enumerate() + { + println!( + "tree_r_last {}/{} inspection shows a {} of base tree root {:?}", + i + 1, + tree_count, + match_str(*cached_root, rebuilt_root), + rebuilt_root + ); + if *cached_root != rebuilt_root { + println!( + "Cached root {:?}, Rebuilt root {:?}", + cached_root, rebuilt_root + ); + } + } + + println!(); + println!( + "CommRLast from p_aux : {:?}", + p_aux.comm_r_last + ); + println!( + "CommRLast [cached tree_r_last root] : {:?}", + tree_r_last_root + ); + println!( + "CommRLast [rebuilt tree_r_last root]: {:?}", + rebuilt_tree_r_last_root + ); + println!(); + println!( + " Cached inspection shows a {} of CommRLast {:?}", + status, tree_r_last_root + ); + println!( + "Rebuilt inspection shows a {} of CommRLast {:?}", + rebuilt_status, rebuilt_tree_r_last_root + ); + Ok(()) } @@ -97,8 +344,58 @@ fn main() -> Result<()> { Arg::with_name("size") .required(true) .long("size") - .default_value("32") - .help("The data size in GiB") + .default_value("34359738368") + .help("The data size in bytes") + .takes_value(true), + ) + .arg( + Arg::with_name("replica") + .long("replica") + .help("The replica file") + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name("cache") + .long("cache") + .help("The cache directory for the output trees") + .required(true) + .takes_value(true), + ); + + let inspect_cmd = SubCommand::with_name("inspect") + .about("Inspect tree_r_last trees and match with p_aux in cache") + .arg( + Arg::with_name("size") + .required(true) + .long("size") + .default_value("34359738368") + .help("The data size in bytes") + .takes_value(true), + ) + .arg( + Arg::with_name("replica") + .long("replica") + .help("The replica file") + .required(true) + .takes_value(true), + ) + .arg( + Arg::with_name("cache") + .long("cache") + .help("The cache directory for the output trees") + .required(true) + .takes_value(true), + ); + + let verify_cmd = SubCommand::with_name("verify") + .about("Verify tree_r_last trees and check for cache mis-match") + .arg( + Arg::with_name("size") + .required(true) + .long("size") + .default_value("34359738368") + .help("The data size in bytes") .takes_value(true), ) .arg( @@ -119,6 +416,8 @@ fn main() -> Result<()> { let matches = App::new("update_tree_r_cache") .version("0.1") .subcommand(rebuild_cmd) + .subcommand(inspect_cmd) + .subcommand(verify_cmd) .get_matches(); match matches.subcommand() { @@ -127,7 +426,21 @@ fn main() -> Result<()> { let replica = value_t!(m, "replica", PathBuf)?; let size = value_t!(m, "size", usize) .expect("could not convert `size` CLI argument to `usize`"); - run_update(size, cache, replica)?; + run_rebuild(size, cache, replica)?; + } + ("inspect", Some(m)) => { + let cache = value_t!(m, "cache", PathBuf)?; + let replica = value_t!(m, "replica", PathBuf)?; + let size = value_t!(m, "size", usize) + .expect("could not convert `size` CLI argument to `usize`"); + run_inspect(size, cache, replica)?; + } + ("verify", Some(m)) => { + let cache = value_t!(m, "cache", PathBuf)?; + let replica = value_t!(m, "replica", PathBuf)?; + let size = value_t!(m, "size", usize) + .expect("could not convert `size` CLI argument to `usize`"); + run_verify(size, cache, replica)?; } _ => panic!("Unrecognized subcommand"), } From e07d14ff9eac6f9dc5b1d47eca7f605ca648f4b3 Mon Sep 17 00:00:00 2001 From: nemo Date: Fri, 19 Jun 2020 08:41:55 -0400 Subject: [PATCH 2/3] feat: re-factor and simplify --- .../src/bin/update_tree_r_cache/main.rs | 149 +++++++----------- filecoin-proofs/src/constants.rs | 35 ++++ 2 files changed, 96 insertions(+), 88 deletions(-) diff --git a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs index 1ba8a79fd..cc9a72906 100644 --- a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs +++ b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs @@ -13,7 +13,9 @@ use typenum::{U0, U8}; use filecoin_proofs::constants::*; use filecoin_proofs::types::*; +use filecoin_proofs::with_shape; use storage_proofs::cache_key::CacheKey; +use storage_proofs::hasher::Hasher; use storage_proofs::merkle::{create_lc_tree, get_base_tree_count, split_config_and_replica}; use storage_proofs::merkle::{LCStore, LCTree, MerkleTreeTrait}; use storage_proofs::util::{default_rows_to_discard, NODE_SIZE}; @@ -23,16 +25,7 @@ fn get_tree_r_info( cache: &PathBuf, replica_path: &PathBuf, ) -> Result<(usize, usize, Vec, ReplicaConfig)> { - let tree_count = match sector_size as u64 { - SECTOR_SIZE_2_KIB => get_base_tree_count::(), - SECTOR_SIZE_4_KIB => get_base_tree_count::(), - SECTOR_SIZE_16_KIB => get_base_tree_count::(), - SECTOR_SIZE_32_KIB => get_base_tree_count::(), - SECTOR_SIZE_512_MIB => get_base_tree_count::(), - SECTOR_SIZE_32_GIB => get_base_tree_count::(), - SECTOR_SIZE_64_GIB => get_base_tree_count::(), - _ => panic!("Unsupported sector size"), - }; + let tree_count = with_shape!(sector_size as u64, get_base_tree_count); // Number of nodes per base tree let base_tree_leafs = sector_size / NODE_SIZE / tree_count; @@ -62,37 +55,45 @@ fn get_tree_r_info( fn get_tree_r_last_root( base_tree_leafs: usize, - sector_size: usize, + sector_size: u64, configs: &[StoreConfig], replica_config: &ReplicaConfig, ) -> Result { let base_tree_len = get_merkle_tree_len(base_tree_leafs, OCT_ARITY)?; - let tree_r_last_root = match sector_size as u64 { - SECTOR_SIZE_2_KIB | SECTOR_SIZE_8_MIB | SECTOR_SIZE_512_MIB => { - ensure!(configs.len() == 1, "Invalid tree-shape specified"); - let store = LCStore::::new_from_disk_with_reader( - base_tree_len, - OCT_ARITY, - &configs[0], - ExternalReader::new_from_path(&replica_config.path)?, - )?; - - let tree_r_last = LCTree::::from_data_store(store, base_tree_leafs)?; - tree_r_last.root() - } - SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_1_GIB => { - let tree_r_last = LCTree::::from_store_configs_and_replica(base_tree_leafs, &configs, &replica_config)?; - tree_r_last.root() - } - SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_GIB => { - let tree_r_last = LCTree::::from_store_configs_and_replica(base_tree_leafs, &configs, &replica_config)?; - tree_r_last.root() - } - SECTOR_SIZE_32_KIB | SECTOR_SIZE_64_GIB => { - let tree_r_last = LCTree::::from_sub_tree_store_configs_and_replica(base_tree_leafs, &configs, &replica_config)?; - tree_r_last.root() - } - _ => panic!("Unsupported sector size"), + let tree_r_last_root = if is_sector_shape_base(sector_size) { + ensure!(configs.len() == 1, "Invalid tree-shape specified"); + let store = LCStore::::new_from_disk_with_reader( + base_tree_len, + OCT_ARITY, + &configs[0], + ExternalReader::new_from_path(&replica_config.path)?, + )?; + + let tree_r_last = SectorShapeBase::from_data_store(store, base_tree_leafs)?; + tree_r_last.root() + } else if is_sector_shape_sub2(sector_size) { + let tree_r_last = SectorShapeSub2::from_store_configs_and_replica( + base_tree_leafs, + &configs, + &replica_config, + )?; + tree_r_last.root() + } else if is_sector_shape_sub8(sector_size) { + let tree_r_last = SectorShapeSub8::from_store_configs_and_replica( + base_tree_leafs, + &configs, + &replica_config, + )?; + tree_r_last.root() + } else if is_sector_shape_top2(sector_size) { + let tree_r_last = SectorShapeTop2::from_sub_tree_store_configs_and_replica( + base_tree_leafs, + &configs, + &replica_config, + )?; + tree_r_last.root() + } else { + panic!("Unsupported sector size"); }; Ok(tree_r_last_root) @@ -114,10 +115,7 @@ fn build_tree_r_last( sector_size: usize, cache: &PathBuf, replica_path: &PathBuf, -) -> Result<( - LCTree, - Vec, -)> { +) -> Result<(::Domain, Vec)> { let (tree_count, base_tree_leafs, configs, replica_config) = get_tree_r_info(sector_size, &cache, &replica_path)?; @@ -164,7 +162,7 @@ fn build_tree_r_last( &replica_config, )?; - Ok((tree_r_last, base_tree_roots)) + Ok((tree_r_last.root(), base_tree_roots)) } fn run_rebuild( @@ -172,53 +170,24 @@ fn run_rebuild( cache: PathBuf, replica_path: PathBuf, ) -> Result<(DefaultTreeDomain, Vec)> { - let (tree_r_last_root, base_tree_roots) = match sector_size as u64 { - SECTOR_SIZE_2_KIB => { - let (tree_r_last, base_tree_roots) = - build_tree_r_last::(sector_size, &cache, &replica_path)?; - (tree_r_last.root(), base_tree_roots) - } - SECTOR_SIZE_4_KIB => { - let (tree_r_last, base_tree_roots) = - build_tree_r_last::(sector_size, &cache, &replica_path)?; - (tree_r_last.root(), base_tree_roots) - } - SECTOR_SIZE_16_KIB => { - let (tree_r_last, base_tree_roots) = - build_tree_r_last::(sector_size, &cache, &replica_path)?; - (tree_r_last.root(), base_tree_roots) - } - SECTOR_SIZE_32_KIB => { - let (tree_r_last, base_tree_roots) = - build_tree_r_last::(sector_size, &cache, &replica_path)?; - (tree_r_last.root(), base_tree_roots) - } - SECTOR_SIZE_512_MIB => { - let (tree_r_last, base_tree_roots) = - build_tree_r_last::(sector_size, &cache, &replica_path)?; - (tree_r_last.root(), base_tree_roots) - } - SECTOR_SIZE_32_GIB => { - let (tree_r_last, base_tree_roots) = - build_tree_r_last::(sector_size, &cache, &replica_path)?; - (tree_r_last.root(), base_tree_roots) - } - SECTOR_SIZE_64_GIB => { - let (tree_r_last, base_tree_roots) = - build_tree_r_last::(sector_size, &cache, &replica_path)?; - (tree_r_last.root(), base_tree_roots) - } - _ => panic!("Unsupported sector size"), - }; - - Ok((tree_r_last_root, base_tree_roots)) + with_shape!( + sector_size as u64, + build_tree_r_last, + sector_size, + &cache, + &replica_path + ) } fn run_inspect(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Result<()> { let (_tree_count, base_tree_leafs, configs, replica_config) = get_tree_r_info(sector_size, &cache, &replica_path)?; - let tree_r_last_root = - get_tree_r_last_root(base_tree_leafs, sector_size, &configs, &replica_config)?; + let tree_r_last_root = get_tree_r_last_root( + base_tree_leafs, + sector_size as u64, + &configs, + &replica_config, + )?; let p_aux = get_persistent_aux(&cache)?; println!("CommRLast from p_aux: {:?}", p_aux.comm_r_last); @@ -229,7 +198,7 @@ fn run_inspect(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Res let status = if tree_r_last_root == p_aux.comm_r_last { "MATCH" } else { - "MIS-MATCH" + "MISMATCH" }; println!("Cached inspection shows a {} of CommRLast", status); @@ -245,7 +214,7 @@ fn run_verify(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu if a == b { "MATCH" } else { - "MIS-MATCH" + "MISMATCH" } }; @@ -262,8 +231,12 @@ fn run_verify(sector_size: usize, cache: PathBuf, replica_path: PathBuf) -> Resu } // Retrieve the tree_r_last root from the cached trees on disk. - let tree_r_last_root = - get_tree_r_last_root(base_tree_leafs, sector_size, &configs, &replica_config)?; + let tree_r_last_root = get_tree_r_last_root( + base_tree_leafs, + sector_size as u64, + &configs, + &replica_config, + )?; // Read comm_r_last from the persistent aux in the cache dir let p_aux: PersistentAux = { diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs index ad95d61c3..b2998bac1 100644 --- a/filecoin-proofs/src/constants.rs +++ b/filecoin-proofs/src/constants.rs @@ -128,6 +128,13 @@ pub type DefaultBinaryTree = storage_proofs::merkle::BinaryMerkleTree; pub type DefaultOctLCTree = storage_proofs::merkle::OctLCMerkleTree; +// Generic shapes +pub type SectorShapeBase = LCTree; +pub type SectorShapeSub2 = LCTree; +pub type SectorShapeSub8 = LCTree; +pub type SectorShapeTop2 = LCTree; + +// Specific shapes pub type SectorShape2KiB = LCTree; pub type SectorShape4KiB = LCTree; pub type SectorShape16KiB = LCTree; @@ -139,6 +146,34 @@ pub type SectorShape1GiB = LCTree; pub type SectorShape32GiB = LCTree; pub type SectorShape64GiB = LCTree; +pub fn is_sector_shape_base(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_2_KIB | SECTOR_SIZE_8_MIB | SECTOR_SIZE_512_MIB => true, + _ => false, + } +} + +pub fn is_sector_shape_sub2(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_1_GIB => true, + _ => false, + } +} + +pub fn is_sector_shape_sub8(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_GIB => true, + _ => false, + } +} + +pub fn is_sector_shape_top2(sector_size: u64) -> bool { + match sector_size { + SECTOR_SIZE_32_KIB | SECTOR_SIZE_64_GIB => true, + _ => false, + } +} + pub use storage_proofs::merkle::{DiskTree, LCTree}; /// Get the correct parameter data for a given cache id. From d507d8bbcb28da7a4baddb916d056abbe5cc5e84 Mon Sep 17 00:00:00 2001 From: nemo Date: Mon, 22 Jun 2020 16:55:59 -0400 Subject: [PATCH 3/3] fix: apply review feedback --- .../src/bin/update_tree_r_cache/main.rs | 8 ++---- filecoin-proofs/src/constants.rs | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs index cc9a72906..0b69b568d 100644 --- a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs +++ b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs @@ -4,12 +4,11 @@ use std::path::{Path, PathBuf}; use anyhow::{ensure, Context, Result}; use bincode::deserialize; use clap::{value_t, App, Arg, SubCommand}; -use generic_array::typenum::{self, Unsigned}; +use generic_array::typenum::Unsigned; use memmap::MmapOptions; use merkletree::merkle::get_merkle_tree_len; use merkletree::store::{ExternalReader, ReplicaConfig, Store, StoreConfig}; use tempdir::TempDir; -use typenum::{U0, U8}; use filecoin_proofs::constants::*; use filecoin_proofs::types::*; @@ -147,10 +146,7 @@ fn build_tree_r_last( (offset + (sector_size / tree_count)), &store_path ); - let tree = LCTree::::from_byte_slice_with_config( - slice, - config.clone(), - )?; + let tree = SectorShapeBase::from_byte_slice_with_config(slice, config.clone())?; base_tree_roots.push(tree.root()); } diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs index b2998bac1..3989e400b 100644 --- a/filecoin-proofs/src/constants.rs +++ b/filecoin-proofs/src/constants.rs @@ -134,17 +134,20 @@ pub type SectorShapeSub2 = LCTree; pub type SectorShapeSub8 = LCTree; pub type SectorShapeTop2 = LCTree; -// Specific shapes -pub type SectorShape2KiB = LCTree; -pub type SectorShape4KiB = LCTree; -pub type SectorShape16KiB = LCTree; -pub type SectorShape32KiB = LCTree; -pub type SectorShape8MiB = LCTree; -pub type SectorShape16MiB = LCTree; -pub type SectorShape512MiB = LCTree; -pub type SectorShape1GiB = LCTree; -pub type SectorShape32GiB = LCTree; -pub type SectorShape64GiB = LCTree; +// Specific size constants by shape +pub type SectorShape2KiB = SectorShapeBase; +pub type SectorShape8MiB = SectorShapeBase; +pub type SectorShape512MiB = SectorShapeBase; + +pub type SectorShape4KiB = SectorShapeSub2; +pub type SectorShape16MiB = SectorShapeSub2; +pub type SectorShape1GiB = SectorShapeSub2; + +pub type SectorShape16KiB = SectorShapeSub8; +pub type SectorShape32GiB = SectorShapeSub8; + +pub type SectorShape32KiB = SectorShapeTop2; +pub type SectorShape64GiB = SectorShapeTop2; pub fn is_sector_shape_base(sector_size: u64) -> bool { match sector_size {