diff --git a/fil-proofs-param/Cargo.toml b/fil-proofs-param/Cargo.toml index 4866cddd57..8977e48794 100644 --- a/fil-proofs-param/Cargo.toml +++ b/fil-proofs-param/Cargo.toml @@ -15,17 +15,17 @@ storage-proofs-post = { path = "../storage-proofs-post", version = "^11.0.0", de storage-proofs-update = { path = "../storage-proofs-update", version = "^11.0.0", default-features = false } filecoin-hashers = { version = "^6.0.0", path = "../filecoin-hashers", default-features = false, features = ["poseidon", "sha256"] } filecoin-proofs = { version = "^11.0.0", path = "../filecoin-proofs", default-features = false } -bitvec = "0.17" +bitvec = "1.0.0" rand = "0.8" lazy_static = "1.2" memmap = "0.7" pbr = "1.0" byteorder = "1" -itertools = "0.9" +itertools = "0.10.3" serde = { version = "1.0", features = ["rc", "derive"] } serde_json = "1.0" ff = "0.11.0" -blake2b_simd = "0.5" +blake2b_simd = "1.0.0" bellperson = "0.18.0" log = "0.4.7" fil_logger = "0.1" @@ -33,32 +33,32 @@ env_proxy = "0.4" flate2 = { version = "1.0.9", features = ["rust_backend"]} tar = "0.4.26" rayon = "1.1.0" -blake2s_simd = "0.5.8" +blake2s_simd = "1.0.0" hex = "0.4.0" merkletree = "0.21.0" bincode = "1.1.2" anyhow = "1.0.23" rand_xorshift = "0.3.0" -sha2 = "0.9.1" +sha2 = "0.10.2" typenum = "1.11.2" gperftools = { version = "0.2", optional = true } generic-array = "0.14.4" structopt = "0.3.12" humansize = "1.1.0" -indicatif = "0.15.0" +indicatif = "0.16.2" group = "0.11.0" -dialoguer = "0.8.0" -clap = "2.33.3" +dialoguer = "0.10.0" +clap = "3.1.6" [dependencies.reqwest] -version = "0.10" +version = "0.11.10" default-features = false features = ["blocking", "native-tls-vendored"] [dev-dependencies] criterion = "0.3" rexpect = "0.4.0" -pretty_assertions = "0.6.1" +pretty_assertions = "1.2.0" failure = "0.1.7" tempfile = "3" diff --git a/fil-proofs-param/src/bin/paramcache.rs b/fil-proofs-param/src/bin/paramcache.rs index a9c18ad246..dc09323d87 100644 --- a/fil-proofs-param/src/bin/paramcache.rs +++ b/fil-proofs-param/src/bin/paramcache.rs @@ -309,7 +309,7 @@ pub fn main() { info!("{}", &message); let spinner = ProgressBar::new_spinner(); - spinner.set_message(&message); + spinner.set_message(message); spinner.enable_steady_tick(100); if opts.only_sector_update { @@ -323,6 +323,6 @@ pub fn main() { } } - spinner.finish_with_message(&format!("✔ {}", &message)); + spinner.finish_with_message(format!("✔ Generated sector size: {}", human_size)); } } diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index 92d5b2b47e..a96e846ed3 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -16,7 +16,7 @@ storage-proofs-post = { path = "../storage-proofs-post", version = "^11.0.0", de storage-proofs-update = { path = "../storage-proofs-update", version = "^11.0.0", default-features = false } filecoin-proofs = { path = "../filecoin-proofs", default-features = false } filecoin-hashers = { path = "../filecoin-hashers", default-features = false, features = ["poseidon", "blake2s", "sha256"] } -clap = "2" +clap = { version = "3.1.6", features = ["derive"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" toml = "0.5" @@ -30,13 +30,13 @@ bellperson = "0.18.0" rand = "0.8" tempfile = "3.0.8" cpu-time = "1.0.0" -git2 = "0.13.6" +git2 = "0.14.2" heim = { git = "https://github.com/heim-rs/heim", rev = "b292f15", features = ["host", "memory", "cpu"] } async-std = "1.6" -blake2s_simd = "0.5.6" +blake2s_simd = "1.0.0" fil_logger = "0.1" log = "0.4.8" -uom = "0.30" +uom = "0.32.0" merkletree = "0.21.0" bincode = "1.1.2" anyhow = "1.0.23" @@ -44,16 +44,17 @@ ff = "0.11.0" rand_xorshift = "0.3.0" bytefmt = "0.1.7" rayon = "1.3.0" -flexi_logger = "0.16.1" +flexi_logger = "0.22.3" typenum = "1.11.2" generic-array = "0.14.4" # `byte-unit = 4.0.13` is the final version that uses Rust 2018 edition. -byte-unit = "=4.0.13" +byte-unit = "4.0.14" fdlimit = "0.2.0" -dialoguer = "0.8.0" +dialoguer = "0.10.0" structopt = "0.3.12" humansize = "1.1.0" blstrs = "0.4.0" +time = "0.3.9" [features] default = ["opencl", "measurements"] @@ -79,4 +80,4 @@ measurements = ["storage-proofs-core/measurements"] profile = ["storage-proofs-core/profile", "measurements"] [target.'cfg(target_arch = "x86_64")'.dependencies] -raw-cpuid = "8.1.2" +raw-cpuid = "10.3.0" diff --git a/fil-proofs-tooling/src/bin/benchy/main.rs b/fil-proofs-tooling/src/bin/benchy/main.rs index b8700cfd0f..363a3ec570 100644 --- a/fil-proofs-tooling/src/bin/benchy/main.rs +++ b/fil-proofs-tooling/src/bin/benchy/main.rs @@ -6,7 +6,7 @@ use std::str::FromStr; use anyhow::Result; use byte_unit::Byte; -use clap::{value_t, App, AppSettings, Arg, SubCommand}; +use clap::{Arg, Command}; use storage_proofs_core::api_version::ApiVersion; @@ -22,52 +22,52 @@ mod winning_post; fn main() -> Result<()> { fil_logger::init(); - let window_post_cmd = SubCommand::with_name("window-post") + let window_post_cmd = Command::new("window-post") .about("Benchmark Window PoST") .arg( - Arg::with_name("preserve-cache") + Arg::new("preserve-cache") .long("preserve-cache") .required(false) .help("Preserve the directory where cached files are persisted") .takes_value(false), ) .arg( - Arg::with_name("skip-precommit-phase1") + Arg::new("skip-precommit-phase1") .long("skip-precommit-phase1") .required(false) .help("Skip precommit phase 1") .takes_value(false), ) .arg( - Arg::with_name("skip-precommit-phase2") + Arg::new("skip-precommit-phase2") .long("skip-precommit-phase2") .required(false) .help("Skip precommit phase 2") .takes_value(false), ) .arg( - Arg::with_name("skip-commit-phase1") + Arg::new("skip-commit-phase1") .long("skip-commit-phase1") .required(false) .help("Skip commit phase 1") .takes_value(false), ) .arg( - Arg::with_name("skip-commit-phase2") + Arg::new("skip-commit-phase2") .long("skip-commit-phase2") .required(false) .help("Skip commit phase 2") .takes_value(false), ) .arg( - Arg::with_name("test-resume") + Arg::new("test-resume") .long("test-resume") .required(false) .help("Test replication resume") .takes_value(false), ) .arg( - Arg::with_name("cache") + Arg::new("cache") .long("cache") .required(false) .help("The directory where cached files are persisted") @@ -75,14 +75,14 @@ fn main() -> Result<()> { .takes_value(true), ) .arg( - Arg::with_name("size") + Arg::new("size") .long("size") .required(true) .help("The data size (e.g. 2KiB)") .takes_value(true), ) .arg( - Arg::with_name("api_version") + Arg::new("api_version") .long("api-version") .required(true) .help("The api_version to use (default: 1.1.0)") @@ -90,23 +90,23 @@ fn main() -> Result<()> { .takes_value(true), ); - let winning_post_cmd = SubCommand::with_name("winning-post") + let winning_post_cmd = Command::new("winning-post") .about("Benchmark Winning PoST") .arg( - Arg::with_name("size") + Arg::new("size") .long("size") .required(true) .help("The data size (e.g. 2KiB)") .takes_value(true), ) .arg( - Arg::with_name("fake") + Arg::new("fake") .long("fake") .help("Use fake replica (default: false)") .takes_value(false), ) .arg( - Arg::with_name("api_version") + Arg::new("api_version") .long("api-version") .required(true) .help("The api_version to use (default: 1.1.0)") @@ -114,23 +114,23 @@ fn main() -> Result<()> { .takes_value(true), ); - let window_post_fake_cmd = SubCommand::with_name("window-post-fake") + let window_post_fake_cmd = Command::new("window-post-fake") .about("Benchmark Window PoST Fake") .arg( - Arg::with_name("size") + Arg::new("size") .long("size") .required(true) .help("The data size (e.g. 2KiB)") .takes_value(true), ) .arg( - Arg::with_name("fake") + Arg::new("fake") .long("fake") .help("Use fake replica (default: false)") .takes_value(false), ) .arg( - Arg::with_name("api_version") + Arg::new("api_version") .long("api-version") .required(true) .help("The api_version to use (default: 1.1.0)") @@ -138,54 +138,54 @@ fn main() -> Result<()> { .takes_value(true), ); - let hash_cmd = SubCommand::with_name("hash-constraints") - .about("Benchmark hash function inside of a circuit"); + let hash_cmd = + Command::new("hash-constraints").about("Benchmark hash function inside of a circuit"); - let prodbench_cmd = SubCommand::with_name("prodbench") + let prodbench_cmd = Command::new("prodbench") .about("Benchmark prodbench") .arg( - Arg::with_name("config") + Arg::new("config") .long("config") .takes_value(true) .required(false) .help("path to config.json"), ) .arg( - Arg::with_name("skip-seal-proof") + Arg::new("skip-seal-proof") .long("skip-seal-proof") .takes_value(false) .help("skip generation (and verification) of seal proof"), ) .arg( - Arg::with_name("skip-post-proof") + Arg::new("skip-post-proof") .long("skip-post-proof") .takes_value(false) .help("skip generation (and verification) of PoSt proof"), ) .arg( - Arg::with_name("only-replicate") + Arg::new("only-replicate") .long("only-replicate") .takes_value(false) .help("only run replication"), ) .arg( - Arg::with_name("only-add-piece") + Arg::new("only-add-piece") .long("only-add-piece") .takes_value(false) .help("only run piece addition"), ); - let merkleproof_cmd = SubCommand::with_name("merkleproofs") + let merkleproof_cmd = Command::new("merkleproofs") .about("Benchmark merkle proof generation") .arg( - Arg::with_name("size") + Arg::new("size") .long("size") .required(true) .help("The data size (e.g. 2KiB)") .takes_value(true), ) .arg( - Arg::with_name("proofs") + Arg::new("proofs") .long("proofs") .default_value("1024") .required(false) @@ -193,7 +193,7 @@ fn main() -> Result<()> { .takes_value(true), ) .arg( - Arg::with_name("validate") + Arg::new("validate") .long("validate") .required(false) .default_value("true") @@ -201,9 +201,9 @@ fn main() -> Result<()> { .takes_value(false), ); - let matches = App::new("benchy") - .setting(AppSettings::ArgRequiredElseHelp) + let matches = Command::new("benchy") .version("0.1") + .arg_required_else_help(true) .subcommand(window_post_cmd) .subcommand(window_post_fake_cmd) .subcommand(winning_post_cmd) @@ -213,7 +213,7 @@ fn main() -> Result<()> { .get_matches(); match matches.subcommand() { - ("window-post", Some(m)) => { + Some(("window-post", m)) => { let preserve_cache = m.is_present("preserve-cache"); // For now these options are combined. let skip_precommit_phase1 = m.is_present("skip-precommit-phase1"); @@ -221,9 +221,9 @@ fn main() -> Result<()> { let skip_commit_phase1 = m.is_present("skip-commit-phase1"); let skip_commit_phase2 = m.is_present("skip-commit-phase2"); let test_resume = m.is_present("test-resume"); - let cache_dir = value_t!(m, "cache", String)?; - let sector_size = Byte::from_str(value_t!(m, "size", String)?)?.get_bytes() as usize; - let api_version = ApiVersion::from_str(&value_t!(m, "api_version", String)?)?; + let cache_dir = m.value_of_t::("cache")?; + let sector_size = Byte::from_str(m.value_of_t::("size")?)?.get_bytes() as usize; + let api_version = ApiVersion::from_str(&m.value_of_t::("api_version")?)?; window_post::run( sector_size, api_version, @@ -236,30 +236,32 @@ fn main() -> Result<()> { test_resume, )?; } - ("winning-post", Some(m)) => { - let sector_size = Byte::from_str(value_t!(m, "size", String)?)?.get_bytes() as usize; + Some(("winning-post", m)) => { + let sector_size = Byte::from_str(m.value_of_t::("size")?)?.get_bytes() as usize; let fake_replica = m.is_present("fake"); - let api_version = ApiVersion::from_str(&value_t!(m, "api_version", String)?)?; + let api_version = ApiVersion::from_str(&m.value_of_t::("api_version")?)?; winning_post::run(sector_size, fake_replica, api_version)?; } - ("window-post-fake", Some(m)) => { - let sector_size = Byte::from_str(value_t!(m, "size", String)?)?.get_bytes() as usize; + Some(("window-post-fake", m)) => { + let sector_size = Byte::from_str(m.value_of_t::("size")?)?.get_bytes() as usize; let fake_replica = m.is_present("fake"); - let api_version = ApiVersion::from_str(&value_t!(m, "api_version", String)?)?; + let api_version = ApiVersion::from_str(&m.value_of_t::("api_version")?)?; window_post_fake::run(sector_size, fake_replica, api_version)?; } - ("hash-constraints", Some(_m)) => { + Some(("hash-constraints", _m)) => { hash_fns::run()?; } - ("merkleproofs", Some(m)) => { - let size = Byte::from_str(value_t!(m, "size", String)?)?.get_bytes() as usize; + Some(("merkleproofs", m)) => { + let size = Byte::from_str(m.value_of_t::("size")?)?.get_bytes() as usize; - let proofs = value_t!(m, "proofs", usize)?; + let proofs = m.value_of_t::("proofs")?; merkleproofs::run(size, proofs, m.is_present("validate"))?; } - ("prodbench", Some(m)) => { + Some(("prodbench", m)) => { let inputs: ProdbenchInputs = if m.is_present("config") { - let file = value_t!(m, "config", String).expect("failed to get config"); + let file = m + .value_of_t::("config") + .expect("failed to get config"); serde_json::from_reader( std::fs::File::open(&file) .unwrap_or_else(|_| panic!("invalid file {:?}", file)), diff --git a/fil-proofs-tooling/src/bin/check_parameters/main.rs b/fil-proofs-tooling/src/bin/check_parameters/main.rs index de24684be4..99773de69e 100644 --- a/fil-proofs-tooling/src/bin/check_parameters/main.rs +++ b/fil-proofs-tooling/src/bin/check_parameters/main.rs @@ -3,7 +3,7 @@ use std::path::Path; use anyhow::Result; use bellperson::groth16::MappedParameters; use blstrs::Bls12; -use clap::{value_t, App, Arg, SubCommand}; +use clap::{Arg, Command}; use storage_proofs_core::parameter_cache::read_cached_params; @@ -14,24 +14,22 @@ fn run_map(parameter_file: &Path) -> Result> { fn main() { fil_logger::init(); - let map_cmd = SubCommand::with_name("map") - .about("build mapped parameters") - .arg( - Arg::with_name("param") - .long("parameter-file") - .help("The parameter file to map") - .required(true) - .takes_value(true), - ); + let map_cmd = Command::new("map").about("build mapped parameters").arg( + Arg::new("param") + .long("parameter-file") + .help("The parameter file to map") + .required(true) + .takes_value(true), + ); - let matches = App::new("check_parameters") + let matches = Command::new("check_parameters") .version("0.1") .subcommand(map_cmd) .get_matches(); match matches.subcommand() { - ("map", Some(m)) => { - let parameter_file_str = value_t!(m, "param", String).expect("param failed"); + Some(("map", m)) => { + let parameter_file_str = m.value_of_t::("param").expect("param failed"); run_map(Path::new(¶meter_file_str)).expect("run_map failed"); } _ => panic!("Unrecognized subcommand"), diff --git a/fil-proofs-tooling/src/bin/gen_graph_cache/main.rs b/fil-proofs-tooling/src/bin/gen_graph_cache/main.rs index 9e7034199d..4589f8fcd3 100644 --- a/fil-proofs-tooling/src/bin/gen_graph_cache/main.rs +++ b/fil-proofs-tooling/src/bin/gen_graph_cache/main.rs @@ -4,7 +4,7 @@ use std::io::BufWriter; use std::path::Path; use anyhow::Result; -use clap::{value_t, App, Arg}; +use clap::{Arg, Command}; use filecoin_hashers::sha256::Sha256Hasher; use filecoin_proofs::{ with_shape, DRG_DEGREE, EXP_DEGREE, SECTOR_SIZE_2_KIB, SECTOR_SIZE_32_GIB, SECTOR_SIZE_512_MIB, @@ -71,17 +71,17 @@ fn gen_graph_cache( fn main() -> Result<()> { fil_logger::init(); - let matches = App::new("gen_graph_cache") + let matches = Command::new("gen_graph_cache") .version("0.1") .about("Generates and/or verifies parent graph cache files") .arg( - Arg::with_name("json") + Arg::new("json") .long("json") .help("Creates a new json output file.") .default_value("false"), ) .arg( - Arg::with_name("size") + Arg::new("size") .long("size") .help("Generate and/or verify the graph cache files for a single sector size") .default_value("0"), @@ -184,8 +184,12 @@ fn main() -> Result<()> { .collect::>(); let mut parent_cache_summary_map: ParentCacheSummaryMap = BTreeMap::new(); - let size = value_t!(matches, "size", u64).expect("failed to get size"); - let json = value_t!(matches, "json", bool).expect("failed to get json"); + let size = matches + .value_of_t::("size") + .expect("failed to get size"); + let json = matches + .value_of_t::("json") + .expect("failed to get json"); if size == 0 { println!( diff --git a/fil-proofs-tooling/src/bin/gpu-cpu-test/main.rs b/fil-proofs-tooling/src/bin/gpu-cpu-test/main.rs index 6660a10d30..1de4a8013a 100644 --- a/fil-proofs-tooling/src/bin/gpu-cpu-test/main.rs +++ b/fil-proofs-tooling/src/bin/gpu-cpu-test/main.rs @@ -3,12 +3,12 @@ use std::collections::HashMap; use std::process::{self, Child, Command, Stdio}; -use std::str; +use std::str::{self, FromStr}; use std::sync::mpsc::{self, Receiver, Sender, TryRecvError}; use std::thread; use std::time::{Duration, Instant}; -use clap::{arg_enum, value_t, App, Arg}; +use clap::Arg; use fil_proofs_tooling::shared::{create_replica, PROVER_ID, RANDOMNESS}; use filecoin_proofs::constants::{SectorShape8MiB, SECTOR_SIZE_8_MIB}; use filecoin_proofs::types::{PoStConfig, SectorSize}; @@ -34,11 +34,24 @@ const POST_CONFIG: PoStConfig = PoStConfig { api_version: FIXED_API_VERSION, }; -arg_enum! { - #[derive(Debug)] - pub enum Mode { - Threads, - Processes, +#[derive(Debug, Clone)] +pub enum Mode { + Threads, + Processes, +} + +impl FromStr for Mode { + type Err = clap::Error; + + fn from_str(s: &str) -> Result { + match s { + "threads" => Ok(Mode::Threads), + "processes" => Ok(Mode::Processes), + _ => Err(clap::Error::raw( + clap::ErrorKind::InvalidValue, + format!("unknown mode '{}'", s), + )), + } } } @@ -57,12 +70,14 @@ pub fn colored_with_thread( write!( writer, "{} {} {} {} {} > {}", - now.now().format("%Y-%m-%dT%H:%M:%S%.3f"), + now.now() + .format(time::macros::format_description!("%Y-%m-%dT%H:%M:%S%.3f")) + .expect("failed to format time"), process::id(), thread::current() .name() .unwrap_or(&format!("{:?}", thread::current().id())), - flexi_logger::style(level, level), + flexi_logger::style(level).paint(level.to_string()), record.module_path().unwrap_or(""), record.args(), ) @@ -227,49 +242,56 @@ fn spawn_process(name: &str, gpu_stealing: bool) -> Child { } fn main() { - flexi_logger::Logger::with_env() + flexi_logger::Logger::try_with_env() + .expect("Initializing logger from env failed") .format(colored_with_thread) .start() .expect("Initializing logger failed. Was another logger already initialized?"); - let matches = App::new("gpu-cpu-test") + let matches = clap::Command::new("gpu-cpu-test") .version("0.1") .about("Tests if moving proofs from GPU to CPU works") .arg( - Arg::with_name("parallel") + Arg::new("parallel") .long("parallel") .help("Run multiple proofs in parallel.") .default_value("3"), ) .arg( - Arg::with_name("gpu-stealing") + Arg::new("gpu-stealing") .long("gpu-stealing") .help("Force high priority proof on the GPU and let low priority one continue on CPU.") .default_value("true"), ) .arg( - Arg::with_name("mode") + Arg::new("mode") .long("mode") .help("Whether to run with threads or processes.") - .possible_values(&["threads", "processes"]) - .case_insensitive(true) - .default_value("threads"), + .possible_values(&["threads", "processes"]) + .ignore_case(true) + .default_value("threads"), ) .get_matches(); - let parallel = value_t!(matches, "parallel", u8).expect("failed to get parallel"); + let parallel = matches + .value_of_t::("parallel") + .expect("failed to get parallel"); if parallel == 1 { info!("Running high priority proof only") } else { info!("Running high and low priority proofs in parallel") } - let gpu_stealing = value_t!(matches, "gpu-stealing", bool).expect("failed to get gpu-stealing"); + let gpu_stealing = matches + .value_of_t::("gpu-stealing") + .expect("failed to get gpu-stealing"); if gpu_stealing { info!("Force low piority proofs to CPU") } else { info!("Let everyone queue up to run on GPU") } - let mode = value_t!(matches, "mode", Mode).unwrap_or_else(|e| e.exit()); + let mode = matches + .value_of_t::("mode") + .unwrap_or_else(|e| e.exit()); match mode { Mode::Threads => info!("Using threads"), Mode::Processes => info!("Using processes"), diff --git a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs index b0876e7ff1..e6e8be9de9 100644 --- a/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs +++ b/fil-proofs-tooling/src/bin/update_tree_r_cache/main.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use anyhow::{ensure, Context, Result}; use bincode::deserialize; -use clap::{value_t, App, Arg, SubCommand}; +use clap::{Arg, Command}; use filecoin_hashers::Hasher; use filecoin_proofs::{ is_sector_shape_base, is_sector_shape_sub2, is_sector_shape_sub8, is_sector_shape_top2, @@ -313,10 +313,10 @@ fn run_verify(sector_size: usize, cache: &Path, replica_path: &Path) -> Result<( fn main() -> Result<()> { fil_logger::init(); - let rebuild_cmd = SubCommand::with_name("rebuild") + let rebuild_cmd = Command::new("rebuild") .about("Rebuild tree_r_last trees from replica") .arg( - Arg::with_name("size") + Arg::new("size") .required(true) .long("size") .default_value("34359738368") @@ -324,24 +324,24 @@ fn main() -> Result<()> { .takes_value(true), ) .arg( - Arg::with_name("replica") + Arg::new("replica") .long("replica") .help("The replica file") .required(true) .takes_value(true), ) .arg( - Arg::with_name("cache") + Arg::new("cache") .long("cache") .help("The cache directory for the output trees") .required(true) .takes_value(true), ); - let inspect_cmd = SubCommand::with_name("inspect") + let inspect_cmd = Command::new("inspect") .about("Inspect tree_r_last trees and match with p_aux in cache") .arg( - Arg::with_name("size") + Arg::new("size") .required(true) .long("size") .default_value("34359738368") @@ -349,24 +349,24 @@ fn main() -> Result<()> { .takes_value(true), ) .arg( - Arg::with_name("replica") + Arg::new("replica") .long("replica") .help("The replica file") .required(true) .takes_value(true), ) .arg( - Arg::with_name("cache") + Arg::new("cache") .long("cache") .help("The cache directory for the output trees") .required(true) .takes_value(true), ); - let verify_cmd = SubCommand::with_name("verify") + let verify_cmd = Command::new("verify") .about("Verify tree_r_last trees and check for cache mis-match") .arg( - Arg::with_name("size") + Arg::new("size") .required(true) .long("size") .default_value("34359738368") @@ -374,21 +374,21 @@ fn main() -> Result<()> { .takes_value(true), ) .arg( - Arg::with_name("replica") + Arg::new("replica") .long("replica") .help("The replica file") .required(true) .takes_value(true), ) .arg( - Arg::with_name("cache") + Arg::new("cache") .long("cache") .help("The cache directory for the output trees") .required(true) .takes_value(true), ); - let matches = App::new("update_tree_r_cache") + let matches = Command::new("update_tree_r_cache") .version("0.1") .subcommand(rebuild_cmd) .subcommand(inspect_cmd) @@ -396,24 +396,27 @@ fn main() -> Result<()> { .get_matches(); match matches.subcommand() { - ("rebuild", Some(m)) => { - let cache = value_t!(m, "cache", PathBuf)?; - let replica = value_t!(m, "replica", PathBuf)?; - let size = value_t!(m, "size", usize) + Some(("rebuild", m)) => { + let cache = m.value_of_t::("cache")?; + let replica = m.value_of_t::("replica")?; + let size = m + .value_of_t::("size") .expect("could not convert `size` CLI argument to `usize`"); run_rebuild(size, cache.as_path(), replica.as_path())?; } - ("inspect", Some(m)) => { - let cache = value_t!(m, "cache", PathBuf)?; - let replica = value_t!(m, "replica", PathBuf)?; - let size = value_t!(m, "size", usize) + Some(("inspect", m)) => { + let cache = m.value_of_t::("cache")?; + let replica = m.value_of_t::("replica")?; + let size = m + .value_of_t::("size") .expect("could not convert `size` CLI argument to `usize`"); run_inspect(size, cache.as_path(), replica.as_path())?; } - ("verify", Some(m)) => { - let cache = value_t!(m, "cache", PathBuf)?; - let replica = value_t!(m, "replica", PathBuf)?; - let size = value_t!(m, "size", usize) + Some(("verify", m)) => { + let cache = m.value_of_t::("cache")?; + let replica = m.value_of_t::("replica")?; + let size = m + .value_of_t::("size") .expect("could not convert `size` CLI argument to `usize`"); run_verify(size, cache.as_path(), replica.as_path())?; } diff --git a/fil-proofs-tooling/src/metadata.rs b/fil-proofs-tooling/src/metadata.rs index c5910bd63e..be125c632d 100644 --- a/fil-proofs-tooling/src/metadata.rs +++ b/fil-proofs-tooling/src/metadata.rs @@ -82,8 +82,8 @@ impl SystemMetadata { { let cpuid = raw_cpuid::CpuId::new(); let processor = cpuid - .get_extended_function_info() - .and_then(|info| info.processor_brand_string().map(|s| s.to_string())) + .get_processor_brand_string() + .map(|s| s.as_str().to_owned()) .unwrap_or_default(); let (base, max) = cpuid .get_processor_frequency_info() diff --git a/filecoin-hashers/Cargo.toml b/filecoin-hashers/Cargo.toml index ad27ab6bd7..287866b164 100644 --- a/filecoin-hashers/Cargo.toml +++ b/filecoin-hashers/Cargo.toml @@ -20,8 +20,8 @@ rand = "0.8.0" neptune = { version = "5.1.0", optional = true, features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } lazy_static = { version = "1.4.0", optional = true } -blake2s_simd = { version = "0.5.11", optional = true } -sha2 = { version = "0.9.2", optional = true } +blake2s_simd = { version = "1.0.0", optional = true } +sha2 = { version = "0.10.2", optional = true } hex = "0.4.2" [features] diff --git a/filecoin-hashers/src/sha256.rs b/filecoin-hashers/src/sha256.rs index 6e2d4d66e1..525af029f1 100644 --- a/filecoin-hashers/src/sha256.rs +++ b/filecoin-hashers/src/sha256.rs @@ -151,10 +151,7 @@ impl HashFunction for Sha256Function { } fn hash2(a: &Sha256Domain, b: &Sha256Domain) -> Sha256Domain { - let hashed = Sha256::new() - .chain(AsRef::<[u8]>::as_ref(a)) - .chain(AsRef::<[u8]>::as_ref(b)) - .finalize(); + let hashed = Sha256::new().chain_update(a).chain_update(b).finalize(); let mut res = Sha256Domain::default(); res.0.copy_from_slice(&hashed[..]); res.trim_to_fr32(); diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index 4fa7950d91..492544352f 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -14,27 +14,27 @@ storage-proofs-porep = { path = "../storage-proofs-porep", version = "^11.0.0", storage-proofs-post = { path = "../storage-proofs-post", version = "^11.0.0", default-features = false } storage-proofs-update = { path = "../storage-proofs-update", version = "^11.0.0", default-features = false } filecoin-hashers = { version = "^6.0.0", path = "../filecoin-hashers", default-features = false, features = ["poseidon", "sha256"] } -bitvec = "0.17" +bitvec = "1.0.0" rand = "0.8" lazy_static = "1.2" memmap = "0.7" byteorder = "1" -itertools = "0.9" +itertools = "0.10.3" serde = { version = "1.0", features = ["rc", "derive"] } serde_json = "1.0" ff = "0.11.0" -blake2b_simd = "0.5" +blake2b_simd = "1.0.0" bellperson = "0.18.0" log = "0.4.7" fil_logger = "0.1" rayon = "1.1.0" -blake2s_simd = "0.5.8" +blake2s_simd = "1.0.0" hex = "0.4.0" merkletree = "0.21.0" bincode = "1.1.2" anyhow = "1.0.23" rand_xorshift = "0.3.0" -sha2 = "0.9.1" +sha2 = "0.10.2" typenum = "1.11.2" gperftools = { version = "0.2", optional = true } generic-array = "0.14.4" @@ -47,7 +47,7 @@ blstrs = "0.4.0" [dev-dependencies] criterion = "0.3" rexpect = "0.4.0" -pretty_assertions = "0.6.1" +pretty_assertions = "1.2.0" failure = "0.1.7" tempfile = "3" diff --git a/fr32/Cargo.toml b/fr32/Cargo.toml index d6a2e9a2e9..a3baee6fb0 100644 --- a/fr32/Cargo.toml +++ b/fr32/Cargo.toml @@ -19,8 +19,8 @@ blstrs = "0.4.0" [dev-dependencies] bitvec = "0.17" criterion = "0.3" -itertools = "0.9" -pretty_assertions = "0.6.1" +itertools = "0.10.3" +pretty_assertions = "1.2.0" rand = "0.8" rand_xorshift = "0.3" diff --git a/sha2raw/Cargo.toml b/sha2raw/Cargo.toml index 8bcdbe4405..ae06f1ebd2 100644 --- a/sha2raw/Cargo.toml +++ b/sha2raw/Cargo.toml @@ -11,8 +11,8 @@ categories = ["cryptography", "no-std"] edition = "2018" [dependencies] -digest = "0.9" -block-buffer = "0.9" +digest = "0.10.3" +block-buffer = "0.10.2" fake-simd = "0.1" opaque-debug = "0.3" sha2-asm = { version = "0.6", optional = true } @@ -22,13 +22,13 @@ byteorder = "1.3.4" version = "1.4.0" [target.'cfg(target_arch = "x86_64")'.dependencies] -cpuid-bool = "0.1.0" +cpufeatures = "0.2.2" [dev-dependencies] -digest = { version = "0.9", features = ["dev", "std"] } -sha2 = "0.9.1" -rand = "0.7.3" -rand_xorshift = "0.2.0" +digest = { version = "0.10.3", features = ["dev", "std"] } +sha2 = "0.10.2" +rand = "0.8.5" +rand_xorshift = "0.3.0" [features] default = ["asm"] diff --git a/sha2raw/src/platform.rs b/sha2raw/src/platform.rs index 8e25f902c9..fa34bf648a 100644 --- a/sha2raw/src/platform.rs +++ b/sha2raw/src/platform.rs @@ -43,7 +43,9 @@ impl Implementation { pub fn sha_if_supported() -> Option { // Use raw_cpuid instead of is_x86_feature_detected, to ensure the check // never happens at compile time. - let is_runtime_ok = cpuid_bool::cpuid_bool!("sha"); + cpufeatures::new!(cpuid_sha, "sha"); + + let is_runtime_ok = cpuid_sha::get(); #[cfg(target_feature = "sha")] { diff --git a/storage-proofs-core/Cargo.toml b/storage-proofs-core/Cargo.toml index 3aa168e38d..c5ab127d2f 100644 --- a/storage-proofs-core/Cargo.toml +++ b/storage-proofs-core/Cargo.toml @@ -16,19 +16,18 @@ filecoin-hashers = { path = "../filecoin-hashers", version = "^6.0.0", default-f rand = "0.8" merkletree = "0.21.0" byteorder = "1" -config = { version = "0.10.1", default-features = false, features = ["toml"] } -itertools = "0.9" +config = { version = "0.12.0", default-features = false, features = ["toml"] } +itertools = "0.10.3" lazy_static = "1.2" memmap = "0.7" -aes = "0.6" -block-modes = "0.7" -sha2 = "0.9.1" +aes = "0.8.1" +sha2 = "0.10.2" tempfile = "3" fs2 = "0.4" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} -blake2b_simd = "0.5" -blake2s_simd = "0.5" +blake2b_simd = "1.0.0" +blake2s_simd = "1.0.0" toml = "0.5" ff = "0.11.0" bellperson = "0.18.0" @@ -43,17 +42,18 @@ neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity1 cpu-time = { version = "1.0", optional = true } gperftools = { version = "0.2", optional = true } num_cpus = "1.10.1" -semver = "0.11.0" +semver = "1.0.6" fr32 = { path = "../fr32", version = "^4.0.0"} pairing = "0.21" blstrs = "0.4.0" +cbc = { version = "0.1.2", features = ["std"] } [dev-dependencies] -proptest = "0.10" +proptest = "1.0.0" criterion = "0.3" -bitvec = "0.17" +bitvec = "1.0.0" rand_xorshift = "0.3.0" -pretty_assertions = "0.6.1" +pretty_assertions = "1.2.0" sha2raw = { path = "../sha2raw", version = "^6.0.0"} filecoin-hashers = { path = "../filecoin-hashers", version = "^6.0.0", default-features = false, features = ["blake2s", "sha256", "poseidon"] } diff --git a/storage-proofs-core/src/crypto/aes.rs b/storage-proofs-core/src/crypto/aes.rs index a0aeca2cc0..a0d3f13a82 100644 --- a/storage-proofs-core/src/crypto/aes.rs +++ b/storage-proofs-core/src/crypto/aes.rs @@ -1,6 +1,7 @@ -use aes::Aes256; +use aes::cipher::block_padding::ZeroPadding; +use aes::cipher::crypto_common::KeyIvInit; +use aes::cipher::{BlockDecryptMut, BlockEncryptMut}; use anyhow::{ensure, Context}; -use block_modes::{block_padding::ZeroPadding, BlockMode, Cbc}; use crate::error::Result; @@ -9,17 +10,19 @@ const IV: [u8; 16] = [0u8; 16]; pub fn encode(key: &[u8], plaintext: &[u8]) -> Result> { ensure!(key.len() == 32, "invalid key length"); - let mode = Cbc::::new_var(key, &IV).context("invalid key")?; + let mode = cbc::Encryptor::::new_from_slices(key, &IV).context("invalid key")?; - Ok(mode.encrypt_vec(plaintext)) + Ok(mode.encrypt_padded_vec_mut::(plaintext)) } pub fn decode(key: &[u8], ciphertext: &[u8]) -> Result> { ensure!(key.len() == 32, "invalid key length"); - let mode = Cbc::::new_var(key, &IV).context("invalid key")?; + let mode = cbc::Decryptor::::new_from_slices(key, &IV).context("invalid key")?; - let res = mode.decrypt_vec(ciphertext).context("failed to decrypt")?; + let res = mode + .decrypt_padded_vec_mut::(ciphertext) + .context("failed to decrypt")?; Ok(res) } diff --git a/storage-proofs-core/src/crypto/mod.rs b/storage-proofs-core/src/crypto/mod.rs index 344d216471..02c7bb3fa9 100644 --- a/storage-proofs-core/src/crypto/mod.rs +++ b/storage-proofs-core/src/crypto/mod.rs @@ -15,8 +15,8 @@ pub fn derive_porep_domain_seed( porep_id: [u8; 32], ) -> [u8; 32] { Sha256::new() - .chain(domain_separation_tag.0) - .chain(porep_id) + .chain_update(domain_separation_tag.0) + .chain_update(porep_id) .finalize() .into() } diff --git a/storage-proofs-core/src/settings.rs b/storage-proofs-core/src/settings.rs index f45dcf5979..b0e865d6c6 100644 --- a/storage-proofs-core/src/settings.rs +++ b/storage-proofs-core/src/settings.rs @@ -92,11 +92,10 @@ impl Settings { fn new() -> Result { set_gpu_framework(); - let mut s = Config::new(); - - s.merge(File::with_name(SETTINGS_PATH).required(false))?; - s.merge(Environment::with_prefix(PREFIX))?; - - s.try_into() + Config::builder() + .add_source(File::with_name(SETTINGS_PATH).required(false)) + .add_source(Environment::with_prefix(PREFIX)) + .build()? + .try_deserialize() } } diff --git a/storage-proofs-porep/Cargo.toml b/storage-proofs-porep/Cargo.toml index 01ac3783f1..34b8ed07fc 100644 --- a/storage-proofs-porep/Cargo.toml +++ b/storage-proofs-porep/Cargo.toml @@ -10,14 +10,14 @@ readme = "README.md" [dependencies] crossbeam = "0.8" -digest = "0.9" +digest = "0.10.3" storage-proofs-core = { path = "../storage-proofs-core", version = "^11.0.0", default-features = false} sha2raw = { path = "../sha2raw", version = "^6.0.0"} filecoin-hashers = { path = "../filecoin-hashers", version = "^6.0.0", default-features = false, features = ["poseidon", "sha256"]} rand = "0.8" merkletree = "0.21.0" mapr = "0.8.0" -num-bigint = "0.2" +num-bigint = "0.4.3" num-traits = "0.2" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} @@ -25,7 +25,7 @@ serde_json = "1.0" ff = "0.11.0" bellperson = "0.18.0" log = "0.4.7" -pretty_assertions = "0.6.1" +pretty_assertions = "1.2.0" generic-array = "0.14.4" anyhow = "1.0.23" neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } @@ -35,7 +35,7 @@ bincode = "1.1.2" byteorder = "1.3.4" lazy_static = "1.2" byte-slice-cast = "1.0.0" -hwloc = { version = "0.3.0", optional = true } +hwloc = { version = "0.5.0", optional = true } libc = "0.2" fdlimit = "0.2.0" fr32 = { path = "../fr32", version = "^4.0.0", default-features = false } @@ -45,9 +45,9 @@ pairing = "0.21" blstrs = "0.4.0" [target."cfg(target_arch = \"aarch64\")".dependencies] -sha2 = { version = "0.9.3", features = ["compress", "asm"] } +sha2 = { version = "0.10.2", features = ["compress", "asm"] } [target."cfg(not(target_arch = \"aarch64\"))".dependencies] -sha2 = { version = "0.9.3", features = ["compress"] } +sha2 = { version = "0.10.2", features = ["compress"] } [dev-dependencies] tempfile = "3" diff --git a/storage-proofs-porep/src/stacked/vanilla/challenges.rs b/storage-proofs-porep/src/stacked/vanilla/challenges.rs index 389c76589f..e5334c007b 100644 --- a/storage-proofs-porep/src/stacked/vanilla/challenges.rs +++ b/storage-proofs-porep/src/stacked/vanilla/challenges.rs @@ -51,9 +51,9 @@ impl LayerChallenges { let j: u32 = ((challenges_count * k as usize) + i) as u32; let hash = Sha256::new() - .chain(replica_id.into_bytes()) - .chain(seed) - .chain(&j.to_le_bytes()) + .chain_update(replica_id.into_bytes()) + .chain_update(seed) + .chain_update(&j.to_le_bytes()) .finalize(); let big_challenge = BigUint::from_bytes_le(hash.as_ref()); diff --git a/storage-proofs-porep/src/stacked/vanilla/params.rs b/storage-proofs-porep/src/stacked/vanilla/params.rs index bb1b03dc19..0ea239a670 100644 --- a/storage-proofs-porep/src/stacked/vanilla/params.rs +++ b/storage-proofs-porep/src/stacked/vanilla/params.rs @@ -764,11 +764,11 @@ pub fn generate_replica_id>( porep_seed: &[u8; 32], ) -> H::Domain { let hash = Sha256::new() - .chain(prover_id) - .chain(§or_id.to_be_bytes()[..]) - .chain(ticket) - .chain(AsRef::<[u8]>::as_ref(&comm_d)) - .chain(porep_seed) + .chain_update(prover_id) + .chain_update(§or_id.to_be_bytes()) + .chain_update(ticket) + .chain_update(&comm_d) + .chain_update(porep_seed) .finalize(); bytes_into_fr_repr_safe(hash.as_ref()).into() diff --git a/storage-proofs-post/Cargo.toml b/storage-proofs-post/Cargo.toml index de02118bf5..6ef74e96c2 100644 --- a/storage-proofs-post/Cargo.toml +++ b/storage-proofs-post/Cargo.toml @@ -15,11 +15,11 @@ rand = "0.8" merkletree = "0.21.0" byteorder = "1" crossbeam = "0.8" -sha2 = "0.9.1" +sha2 = "0.10.2" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} -blake2b_simd = "0.5" -blake2s_simd = "0.5" +blake2b_simd = "1.0.0" +blake2s_simd = "1.0.0" ff = "0.11.0" bellperson = "0.18.0" log = "0.4.7" @@ -33,7 +33,7 @@ blstrs = "0.4.0" [dev-dependencies] tempfile = "3" -pretty_assertions = "0.6.1" +pretty_assertions = "1.2.0" filecoin-hashers = { path = "../filecoin-hashers", version = "^6.0.0", default-features = false, features = ["poseidon", "sha256", "blake2s"]} rand_xorshift = "0.3.0" diff --git a/storage-proofs-update/Cargo.toml b/storage-proofs-update/Cargo.toml index dd07209273..5279fbcd8e 100644 --- a/storage-proofs-update/Cargo.toml +++ b/storage-proofs-update/Cargo.toml @@ -10,7 +10,7 @@ readme = "README.md" [dependencies] crossbeam = "0.8" -digest = "0.9" +digest = "0.10.3" storage-proofs-core = { path = "../storage-proofs-core", version = "^11.0.0", default-features = false} storage-proofs-porep = { path = "../storage-proofs-porep", version = "^11.0.0", default-features = false} sha2raw = { path = "../sha2raw", version = "^6.0.0"} @@ -18,7 +18,7 @@ filecoin-hashers = { path = "../filecoin-hashers", version = "^6.0.0", default-f rand = "0.8" merkletree = "0.21.0" mapr = "0.8.0" -num-bigint = "0.2" +num-bigint = "0.4.3" num-traits = "0.2" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} @@ -27,7 +27,7 @@ ff = "0.11.0" bellperson = "0.18.0" blstrs = "0.4.0" log = "0.4.7" -pretty_assertions = "0.6.1" +pretty_assertions = "1.2.0" generic-array = "0.14.4" anyhow = "1.0.23" neptune = { version = "5.1.0", features = ["arity2", "arity4", "arity8", "arity11", "arity16", "arity24", "arity36"] } @@ -37,7 +37,7 @@ bincode = "1.1.2" byteorder = "1.3.4" lazy_static = "1.2" byte-slice-cast = "1.0.0" -hwloc = { version = "0.3.0", optional = true } +hwloc = { version = "0.5.0", optional = true } libc = "0.2" fdlimit = "0.2.0" fr32 = { path = "../fr32", version = "^4.0.0", default-features = false } @@ -46,9 +46,9 @@ fil_logger = "0.1" memmap = "0.7" [target."cfg(target_arch = \"aarch64\")".dependencies] -sha2 = { version = "0.9.3", features = ["compress", "asm"] } +sha2 = { version = "0.10.2", features = ["compress", "asm"] } [target."cfg(not(target_arch = \"aarch64\"))".dependencies] -sha2 = { version = "0.9.3", features = ["compress"] } +sha2 = { version = "0.10.2", features = ["compress"] } [dev-dependencies] tempfile = "3"