diff --git a/.circleci/config.yml b/.circleci/config.yml index 260aa9143..4271950dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,7 @@ jobs: - cargo-v28-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - run: rustup install $(cat rust-toolchain) - run: rustup default $(cat rust-toolchain) + - run: rustup install nightly - run: rustup component add rustfmt-preview - run: rustup component add clippy - run: cargo update @@ -127,9 +128,10 @@ jobs: RUST_TEST_THREADS: 1 no_output_timeout: 30m - # Running with `use_fil_blst=true` should be integrated directly into the test code. For now we - # just re-run the tests that exercise the fil-blst code path with that setting set. - test_fil_blst: + + # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we + # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set. + test_multicore_sdr: docker: - image: filecoin/rust:latest working_directory: /mnt/crate @@ -144,19 +146,28 @@ jobs: - cargo-v28-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - restore_parameter_cache - run: - name: Test with fil-blst enabled + name: Test with use_multicore_sdr pairing enabled command: | ulimit -n 20000 ulimit -u 20000 ulimit -n 20000 - cargo +$(cat rust-toolchain) test --verbose --release --test api -- --ignored + cargo +nightly -Zpackage-features test --all --verbose --release --test api -- --ignored lifecycle environment: RUST_TEST_THREADS: 1 - FIL_PROOFS_USE_FIL_BLST: true + FIL_PROOFS_USE_MULTICORE_SDR: true - # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we - # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set. - test_multicore_sdr: + - run: + name: Test with use_multicore_sdr and blst enabled + command: | + ulimit -n 20000 + ulimit -u 20000 + ulimit -n 20000 + cargo +nightly -Zpackage-features test --all --no-default-features --features gpu,blst --verbose --release --test api -- --ignored lifecycle + environment: + RUST_TEST_THREADS: 1 + FIL_PROOFS_USE_MULTICORE_SDR: true + + test_blst: docker: - image: filecoin/rust:latest working_directory: /mnt/crate @@ -171,15 +182,23 @@ jobs: - cargo-v28-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }} - restore_parameter_cache - run: - name: Test with use_multicore_sdr enabled + name: Test ignored with blst enabled command: | ulimit -n 20000 ulimit -u 20000 ulimit -n 20000 - cargo +$(cat rust-toolchain) test --verbose --release -- --ignored lifecycle + cargo +nightly -Zpackage-features test --all --no-default-features --features gpu,blst --verbose --release --test api -- --ignored environment: RUST_TEST_THREADS: 1 - FIL_PROOFS_USE_MULTICORE_SDR: true + + - run: + name: Test with blst enabled + command: | + ulimit -n 20000 + ulimit -u 20000 + ulimit -n 20000 + cargo +nightly -Zpackage-features test --all --no-default-features --features gpu,blst --verbose + bench: docker: @@ -415,11 +434,12 @@ workflows: requires: - cargo_fetch - ensure_groth_parameters_and_keys_linux - - test_fil_blst: + + - test_multicore_sdr: requires: - cargo_fetch - ensure_groth_parameters_and_keys_linux - - test_multicore_sdr: + - test_blst: requires: - cargo_fetch - ensure_groth_parameters_and_keys_linux diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml index 63f259999..a56d1f537 100644 --- a/fil-proofs-tooling/Cargo.toml +++ b/fil-proofs-tooling/Cargo.toml @@ -10,6 +10,8 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs" readme = "README.md" [dependencies] +storage-proofs = { path = "../storage-proofs", default-features = false } +filecoin-proofs = { path = "../filecoin-proofs", default-features = false } clap = "2" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -20,20 +22,17 @@ regex = "1.3.7" commandspec = "0.12.2" chrono = { version = "0.4.7", features = ["serde"] } memmap = "0.7.0" -bellperson = "0.9.1" -paired = "0.20.0" +bellperson = { version = "0.11", default-features = false } rand = "0.7" -storage-proofs = { path = "../storage-proofs"} -filecoin-proofs = { path = "../filecoin-proofs"} tempfile = "3.0.8" cpu-time = "1.0.0" git2 = "0.13.6" -heim = { version = "0.1.0-beta.1", features = ["host", "memory", "cpu"] } +heim = { git = "https://github.com/heim-rs/heim", rev = "e22e235", features = ["host", "memory", "cpu"] } async-std = "1.6" blake2s_simd = "0.5.6" fil_logger = "0.1" log = "0.4.8" -uom = "0.28" +uom = "0.30" merkletree = "0.21.0" bincode = "1.1.2" anyhow = "1.0.23" @@ -41,9 +40,9 @@ ff = { version = "0.2.3", package = "fff" } rand_xorshift = "0.2.0" bytefmt = "0.1.7" rayon = "1.3.0" -flexi_logger = "0.14.7" +flexi_logger = "0.16.1" typenum = "1.11.2" -generic-array = "0.13.2" +generic-array = "0.14.4" byte-unit = "4.0.9" [features] @@ -51,7 +50,8 @@ default = ["gpu", "measurements"] gpu = ["storage-proofs/gpu", "filecoin-proofs/gpu", "bellperson/gpu"] measurements = ["storage-proofs/measurements"] profile = ["storage-proofs/profile", "measurements"] - +pairing = ["storage-proofs/pairing", "filecoin-proofs/pairing", "bellperson/pairing"] +blst = ["storage-proofs/blst", "filecoin-proofs/blst", "bellperson/blst"] [target.'cfg(target_arch = "x86_64")'.dependencies] -raw-cpuid = "7.0.3" +raw-cpuid = "8.1.2" diff --git a/fil-proofs-tooling/src/bin/benchy/hash_fns.rs b/fil-proofs-tooling/src/bin/benchy/hash_fns.rs index 06be36d2a..545d18aa7 100644 --- a/fil-proofs-tooling/src/bin/benchy/hash_fns.rs +++ b/fil-proofs-tooling/src/bin/benchy/hash_fns.rs @@ -1,8 +1,8 @@ +use bellperson::bls::Bls12; use bellperson::gadgets::boolean::Boolean; use bellperson::util_cs::test_cs::TestConstraintSystem; use bellperson::ConstraintSystem; use fil_proofs_tooling::metadata::Metadata; -use paired::bls12_381::Bls12; use rand::RngCore; use serde::Serialize; use storage_proofs::util::{bits_to_bytes, bytes_into_boolean_vec, bytes_into_boolean_vec_be}; diff --git a/fil-proofs-tooling/src/bin/benchy/prodbench.rs b/fil-proofs-tooling/src/bin/benchy/prodbench.rs index 3d5038467..095d85d50 100644 --- a/fil-proofs-tooling/src/bin/benchy/prodbench.rs +++ b/fil-proofs-tooling/src/bin/benchy/prodbench.rs @@ -1,3 +1,4 @@ +use bellperson::bls::Bls12; use bellperson::util_cs::bench_cs::BenchCS; use bellperson::Circuit; use fil_proofs_tooling::shared::{create_replicas, PROVER_ID, RANDOMNESS, TICKET_BYTES}; @@ -11,7 +12,6 @@ use filecoin_proofs::{ validate_cache_for_commit, PoRepConfig, }; use log::info; -use paired::bls12_381::Bls12; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use serde::{Deserialize, Serialize}; diff --git a/fil-proofs-tooling/src/bin/check_parameters/main.rs b/fil-proofs-tooling/src/bin/check_parameters/main.rs index bd1ffb6ae..d6ae7e5f1 100644 --- a/fil-proofs-tooling/src/bin/check_parameters/main.rs +++ b/fil-proofs-tooling/src/bin/check_parameters/main.rs @@ -1,9 +1,9 @@ use std::path::PathBuf; use anyhow::Result; +use bellperson::bls::Bls12; use bellperson::groth16::MappedParameters; use clap::{value_t, App, Arg, SubCommand}; -use paired::bls12_381::Bls12; use storage_proofs::parameter_cache::read_cached_params; diff --git a/filecoin-proofs/Cargo.toml b/filecoin-proofs/Cargo.toml index 9a88df9d9..dddaf634d 100644 --- a/filecoin-proofs/Cargo.toml +++ b/filecoin-proofs/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs" readme = "README.md" [dependencies] -storage-proofs = { version = "^5.0.0", path = "../storage-proofs" } +storage-proofs = { version = "^5.0.0", path = "../storage-proofs", default-features = false } bitvec = "0.17" chrono = "0.4" rand = "0.7" @@ -23,8 +23,7 @@ serde_json = "1.0" regex = "1.3.7" ff = { version = "0.2.3", package = "fff" } blake2b_simd = "0.5" -bellperson = "0.9.1" -paired = "0.20.0" +bellperson = { version = "0.11", default-features = false } clap = "2" log = "0.4.7" fil_logger = "0.1" @@ -42,14 +41,14 @@ sha2 = "0.9.1" typenum = "1.11.2" bitintr = "0.3.0" gperftools = { version = "0.2", optional = true } -phase2 = { version = "0.8.0", package = "phase21" } +phase2 = { version = "0.10", package = "phase21", default-features = false } simplelog = "0.8.0" rand_chacha = "0.2.1" -dialoguer = "0.6.2" -generic-array = "0.13.2" +dialoguer = "0.7.1" +generic-array = "0.14.4" structopt = "0.3.12" humansize = "1.1.0" -indicatif = "0.14.0" +indicatif = "0.15.0" groupy = "0.3.0" [dependencies.reqwest] @@ -65,12 +64,14 @@ failure = "0.1.7" tempfile = "3" [features] -default = ["gpu"] +default = ["gpu", "pairing"] cpu-profile = ["gperftools"] heap-profile = ["gperftools/heap"] simd = ["storage-proofs/simd"] asm = ["storage-proofs/asm"] gpu = ["storage-proofs/gpu", "bellperson/gpu"] +pairing = ["storage-proofs/pairing", "bellperson/pairing", "phase2/pairing"] +blst = ["storage-proofs/blst", "bellperson/blst", "phase2/blst"] [[bench]] name = "preprocessing" diff --git a/filecoin-proofs/src/api/mod.rs b/filecoin-proofs/src/api/mod.rs index 53b545dae..bb96fcd2d 100644 --- a/filecoin-proofs/src/api/mod.rs +++ b/filecoin-proofs/src/api/mod.rs @@ -587,8 +587,8 @@ where mod tests { use super::*; + use bellperson::bls::Fr; use ff::Field; - use paired::bls12_381::Fr; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs::fr32::bytes_into_fr; diff --git a/filecoin-proofs/src/api/post.rs b/filecoin-proofs/src/api/post.rs index 258120c4b..6afc3c7b5 100644 --- a/filecoin-proofs/src/api/post.rs +++ b/filecoin-proofs/src/api/post.rs @@ -19,7 +19,6 @@ use storage_proofs::post::fallback; use storage_proofs::post::fallback::SectorProof; use storage_proofs::proof::ProofScheme; use storage_proofs::sector::*; -use storage_proofs::settings; use storage_proofs::util::default_rows_to_discard; use crate::api::util::{as_safe_commitment, get_base_tree_leafs, get_base_tree_size}; @@ -624,22 +623,7 @@ pub fn verify_winning_post( k: None, }; - let use_fil_blst = settings::SETTINGS.use_fil_blst; - - let is_valid = if use_fil_blst { - info!("verify_winning_post: use_fil_blst=true"); - let verifying_key_path = post_config.get_cache_verifying_key_path::()?; - fallback::FallbackPoStCompound::verify_blst( - &pub_params, - &pub_inputs, - &proof, - proof.len() / 192, - &fallback::ChallengeRequirements { - minimum_challenge_count: post_config.challenge_count * post_config.sector_count, - }, - &verifying_key_path, - )? - } else { + let is_valid = { let verifying_key = get_post_verifying_key::(&post_config)?; let single_proof = MultiProof::new_from_reader(None, &proof[..], &verifying_key)?; @@ -997,22 +981,7 @@ pub fn verify_window_post( k: None, }; - let use_fil_blst = settings::SETTINGS.use_fil_blst; - - let is_valid = if use_fil_blst { - info!("verify_window_post: use_fil_blst=true"); - let verifying_key_path = post_config.get_cache_verifying_key_path::()?; - fallback::FallbackPoStCompound::verify_blst( - &pub_params, - &pub_inputs, - &proof, - proof.len() / 192, - &fallback::ChallengeRequirements { - minimum_challenge_count: post_config.challenge_count * post_config.sector_count, - }, - &verifying_key_path, - )? - } else { + let is_valid = { let verifying_key = get_post_verifying_key::(&post_config)?; let multi_proof = MultiProof::new_from_reader(partitions, &proof[..], &verifying_key)?; diff --git a/filecoin-proofs/src/api/seal.rs b/filecoin-proofs/src/api/seal.rs index 1a4a46969..8dab0b715 100644 --- a/filecoin-proofs/src/api/seal.rs +++ b/filecoin-proofs/src/api/seal.rs @@ -3,11 +3,11 @@ use std::io::prelude::*; use std::path::{Path, PathBuf}; use anyhow::{ensure, Context, Result}; +use bellperson::bls::Fr; use bincode::{deserialize, serialize}; use log::{info, trace}; use memmap::MmapOptions; use merkletree::store::{DiskStore, Store, StoreConfig}; -use paired::bls12_381::Fr; use storage_proofs::cache_key::CacheKey; use storage_proofs::compound_proof::{self, CompoundProof}; use storage_proofs::drgraph::Graph; @@ -21,7 +21,6 @@ use storage_proofs::porep::stacked::{ }; use storage_proofs::proof::ProofScheme; use storage_proofs::sector::SectorId; -use storage_proofs::settings; use storage_proofs::util::default_rows_to_discard; use crate::api::util::{ @@ -505,7 +504,7 @@ pub fn seal_commit_phase2( )?; info!("snark_proof:finish"); - let proof = MultiProof::new(groth_proofs, &groth_params.vk); + let proof = MultiProof::new(groth_proofs, &groth_params.pvk); let mut buf = Vec::with_capacity( SINGLE_PARTITION_PROOF_LEN * usize::from(PoRepProofPartitions::from(porep_config)), @@ -609,27 +608,7 @@ pub fn verify_seal( k: None, }; - let use_fil_blst = settings::SETTINGS.use_fil_blst; - - let result = if use_fil_blst { - info!("verify_seal: use_fil_blst=true"); - let verifying_key_path = porep_config.get_cache_verifying_key_path::()?; - - StackedCompound::verify_blst( - &compound_public_params, - &public_inputs, - &proof_vec, - proof_vec.len() / 192, - &ChallengeRequirements { - minimum_challenges: *POREP_MINIMUM_CHALLENGES - .read() - .expect("POREP_MINIMUM_CHALLENGES poisoned") - .get(&u64::from(SectorSize::from(porep_config))) - .expect("unknown sector size") as usize, - }, - &verifying_key_path, - ) - } else { + let result = { let sector_bytes = PaddedBytesAmount::from(porep_config); let verifying_key = get_stacked_verifying_key::(porep_config)?; diff --git a/filecoin-proofs/src/api/util.rs b/filecoin-proofs/src/api/util.rs index e63914227..e310d3de4 100644 --- a/filecoin-proofs/src/api/util.rs +++ b/filecoin-proofs/src/api/util.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Result}; +use bellperson::bls::Fr; use merkletree::merkle::{get_merkle_tree_leafs, get_merkle_tree_len}; -use paired::bls12_381::Fr; use storage_proofs::fr32::{bytes_into_fr, fr_into_bytes}; use storage_proofs::hasher::{Domain, Hasher}; use storage_proofs::merkle::{get_base_tree_count, MerkleTreeTrait}; diff --git a/filecoin-proofs/src/bin/circuitinfo.rs b/filecoin-proofs/src/bin/circuitinfo.rs index 7914eac26..40fcf461f 100644 --- a/filecoin-proofs/src/bin/circuitinfo.rs +++ b/filecoin-proofs/src/bin/circuitinfo.rs @@ -4,7 +4,7 @@ use log::{info, warn}; use structopt::StructOpt; use bellperson::util_cs::bench_cs::BenchCS; -use bellperson::Circuit; +use bellperson::{bls::Bls12, Circuit}; use filecoin_proofs::constants::*; use filecoin_proofs::parameters::{ public_params, window_post_public_params, winning_post_public_params, @@ -12,7 +12,6 @@ use filecoin_proofs::parameters::{ use filecoin_proofs::types::*; use filecoin_proofs::with_shape; use filecoin_proofs::PoStType; -use paired::bls12_381::Bls12; use storage_proofs::compound_proof::CompoundProof; use storage_proofs::porep::stacked::{StackedCompound, StackedDrg}; use storage_proofs::post::fallback::{FallbackPoSt, FallbackPoStCircuit, FallbackPoStCompound}; diff --git a/filecoin-proofs/src/bin/phase2.rs b/filecoin-proofs/src/bin/phase2.rs index af3353cb2..5055c6cf3 100644 --- a/filecoin-proofs/src/bin/phase2.rs +++ b/filecoin-proofs/src/bin/phase2.rs @@ -9,6 +9,7 @@ use std::sync::mpsc::{channel, TryRecvError}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; +use bellperson::bls::{Bls12, G1Affine, G1Uncompressed, G2Affine, G2Uncompressed}; use bellperson::groth16; use byteorder::{BigEndian, ReadBytesExt}; use clap::{App, AppSettings, Arg, ArgGroup, SubCommand}; @@ -22,7 +23,6 @@ use filecoin_proofs::types::{ use filecoin_proofs::with_shape; use groupy::{CurveAffine, EncodedPoint}; use log::{error, info, warn}; -use paired::bls12_381::{Bls12, G1Affine, G1Uncompressed, G2Affine, G2Uncompressed}; use phase2::small::{read_small_params_from_large_file, MPCSmall, Streamer}; use phase2::MPCParameters; use rand::rngs::OsRng; diff --git a/filecoin-proofs/src/caches.rs b/filecoin-proofs/src/caches.rs index e2698f6e8..1ad4580c2 100644 --- a/filecoin-proofs/src/caches.rs +++ b/filecoin-proofs/src/caches.rs @@ -3,10 +3,10 @@ use std::sync::Arc; use std::sync::Mutex; use anyhow::Result; +use bellperson::bls::Bls12; use bellperson::groth16; use lazy_static::lazy_static; use log::info; -use paired::bls12_381::Bls12; use storage_proofs::compound_proof::CompoundProof; use storage_proofs::porep::stacked::{StackedCompound, StackedDrg}; use storage_proofs::post::fallback; @@ -16,11 +16,11 @@ use crate::parameters::{public_params, window_post_public_params, winning_post_p use crate::types::*; type Bls12GrothParams = groth16::MappedParameters; -pub type Bls12VerifyingKey = groth16::VerifyingKey; +pub type Bls12PreparedVerifyingKey = groth16::PreparedVerifyingKey; type Cache = HashMap>; type GrothMemCache = Cache; -type VerifyingKeyMemCache = Cache; +type VerifyingKeyMemCache = Cache; lazy_static! { static ref GROTH_PARAM_MEMORY_CACHE: Mutex = Default::default(); @@ -67,9 +67,12 @@ where } #[inline] -pub fn lookup_verifying_key(identifier: String, generator: F) -> Result> +pub fn lookup_verifying_key( + identifier: String, + generator: F, +) -> Result> where - F: FnOnce() -> Result, + F: FnOnce() -> Result, { let vk_identifier = format!("{}-verifying-key", &identifier); cache_lookup(&*VERIFYING_KEY_MEMORY_CACHE, vk_identifier, generator) @@ -148,7 +151,7 @@ pub fn get_post_params( pub fn get_stacked_verifying_key( porep_config: PoRepConfig, -) -> Result> { +) -> Result> { let public_params = public_params( PaddedBytesAmount::from(porep_config), usize::from(PoRepProofPartitions::from(porep_config)), @@ -156,11 +159,11 @@ pub fn get_stacked_verifying_key( )?; let vk_generator = || { - as CompoundProof< + let vk = as CompoundProof< StackedDrg, _, - >>::verifying_key::(None, &public_params) - .map_err(Into::into) + >>::verifying_key::(None, &public_params)?; + Ok(bellperson::groth16::prepare_verifying_key(&vk)) }; Ok(lookup_verifying_key( @@ -174,17 +177,19 @@ pub fn get_stacked_verifying_key( pub fn get_post_verifying_key( post_config: &PoStConfig, -) -> Result> { +) -> Result> { match post_config.typ { PoStType::Winning => { let post_public_params = winning_post_public_params::(post_config)?; let vk_generator = || { - as CompoundProof< + let vk = as CompoundProof< fallback::FallbackPoSt, fallback::FallbackPoStCircuit, - >>::verifying_key::(None, &post_public_params) - .map_err(Into::into) + >>::verifying_key::( + None, &post_public_params + )?; + Ok(bellperson::groth16::prepare_verifying_key(&vk)) }; Ok(lookup_verifying_key( @@ -199,11 +204,13 @@ pub fn get_post_verifying_key( let post_public_params = window_post_public_params::(post_config)?; let vk_generator = || { - as CompoundProof< + let vk = as CompoundProof< fallback::FallbackPoSt, fallback::FallbackPoStCircuit, - >>::verifying_key::(None, &post_public_params) - .map_err(Into::into) + >>::verifying_key::( + None, &post_public_params + )?; + Ok(bellperson::groth16::prepare_verifying_key(&vk)) }; Ok(lookup_verifying_key( diff --git a/filecoin-proofs/src/pieces.rs b/filecoin-proofs/src/pieces.rs index 5da6d03f9..a9bcb9c34 100644 --- a/filecoin-proofs/src/pieces.rs +++ b/filecoin-proofs/src/pieces.rs @@ -357,7 +357,7 @@ mod tests { use crate::constants::{DRG_DEGREE, EXP_DEGREE}; use crate::types::DataTree; - use paired::bls12_381::Fr; + use bellperson::bls::Fr; use rand::{Rng, RngCore, SeedableRng}; use rand_xorshift::XorShiftRng; use storage_proofs::drgraph::Graph; diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index e45adfb84..2d0aa9f04 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -5,8 +5,8 @@ use std::path::{Path, PathBuf}; use std::sync::Once; use anyhow::Result; +use bellperson::bls::Fr; use ff::Field; -use paired::bls12_381::Fr; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; use storage_proofs::hasher::Hasher; diff --git a/rust-fil-proofs.config.toml.sample b/rust-fil-proofs.config.toml.sample index cb68e9700..f2106e8dd 100644 --- a/rust-fil-proofs.config.toml.sample +++ b/rust-fil-proofs.config.toml.sample @@ -31,8 +31,5 @@ rows_to_discard = 2 # This value is defaulted to the number of cores available on your system. #window_post_synthesis_num_cpus = 8 -# This enables accelerate snark verification -use_fil_blst = false - # This enables multicore SDR replication use_multicore_sdr = false diff --git a/sha2raw/Cargo.toml b/sha2raw/Cargo.toml index 69d21b1ac..caba9ad1b 100644 --- a/sha2raw/Cargo.toml +++ b/sha2raw/Cargo.toml @@ -12,10 +12,11 @@ edition = "2018" [dependencies] digest = "0.9" -block-buffer = "0.7" +block-buffer = "0.9" fake-simd = "0.1" -opaque-debug = "0.2" +opaque-debug = "0.3" sha2-asm = { version = "0.5", optional = true } +byteorder = "1.3.4" [dependencies.lazy_static] version = "1.4.0" diff --git a/sha2raw/src/sha256.rs b/sha2raw/src/sha256.rs index 9130fdea8..b792bcb33 100644 --- a/sha2raw/src/sha256.rs +++ b/sha2raw/src/sha256.rs @@ -1,4 +1,4 @@ -use block_buffer::byteorder::{ByteOrder, BE}; +use byteorder::{ByteOrder, BE}; use crate::consts::H256; use crate::platform::Implementation; @@ -79,7 +79,7 @@ impl Sha256 { } } -opaque_debug::impl_opaque_debug!(Sha256); +opaque_debug::implement!(Sha256); #[cfg(test)] mod tests { diff --git a/sha2raw/src/sha256_utils.rs b/sha2raw/src/sha256_utils.rs index 0e6d8a743..5f55cd783 100644 --- a/sha2raw/src/sha256_utils.rs +++ b/sha2raw/src/sha256_utils.rs @@ -283,7 +283,7 @@ fn sha256_digest_block_u32(state: &mut [u32; 8], block: &[u32; 16]) { #[inline] pub fn compress256(state: &mut [u32; 8], blocks: &[&[u8]]) { use crate::consts::BLOCK_LEN; - use block_buffer::byteorder::{ByteOrder, BE}; + use byteorder::{ByteOrder, BE}; let mut block_u32 = [0u32; BLOCK_LEN]; diff --git a/storage-proofs/Cargo.toml b/storage-proofs/Cargo.toml index 22de6d12a..cbcc3995c 100644 --- a/storage-proofs/Cargo.toml +++ b/storage-proofs/Cargo.toml @@ -9,16 +9,17 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs" readme = "README.md" [dependencies] -storage-proofs-core = { path = "./core", version = "^5.0.0"} -storage-proofs-post = { path = "./post", version = "^5.0.0"} -storage-proofs-porep = { path = "./porep", version = "^5.0.0"} +storage-proofs-core = { path = "./core", version = "^5.0.0", default-features = false } +storage-proofs-post = { path = "./post", version = "^5.0.0", default-features = false } +storage-proofs-porep = { path = "./porep", version = "^5.0.0", default-features = false } [features] -default = ["gpu"] +default = ["gpu", "pairing"] simd = ["storage-proofs-core/simd"] asm = ["storage-proofs-core/asm"] -gpu = ["storage-proofs-core/gpu"] +gpu = ["storage-proofs-core/gpu", "storage-proofs-porep/gpu", "storage-proofs-post/gpu"] measurements = ["storage-proofs-core/measurements"] profile = ["measurements"] - +pairing = ["storage-proofs-core/pairing", "storage-proofs-post/pairing", "storage-proofs-porep/pairing"] +blst = ["storage-proofs-core/blst", "storage-proofs-post/blst", "storage-proofs-porep/blst"] diff --git a/storage-proofs/core/Cargo.toml b/storage-proofs/core/Cargo.toml index 6083013ad..a2d0a9eda 100644 --- a/storage-proofs/core/Cargo.toml +++ b/storage-proofs/core/Cargo.toml @@ -19,8 +19,8 @@ config = { version = "0.10.1", default-features = false, features = ["toml"] } itertools = "0.9" lazy_static = "1.2" memmap = "0.7" -aes = "0.3" -block-modes = "0.3" +aes = "0.6" +block-modes = "0.7" sha2 = "0.9.1" tempfile = "3" fs2 = "0.4" @@ -30,20 +30,18 @@ blake2b_simd = "0.5" blake2s_simd = "0.5" toml = "0.5" ff = { version = "0.2.3", package = "fff" } -bellperson = "0.9.1" -paired = { version = "0.20.0", features = ["serde"] } +bellperson = { version = "0.11", default-features = false } serde_json = "1.0" log = "0.4.7" rand_chacha = "0.2.1" hex = "0.4.0" -generic-array = "0.13.2" +generic-array = "0.14.4" anyhow = "1.0.23" thiserror = "1.0.6" -neptune = { version = "=1.2.1", features = ["gpu"] } +neptune = { version = "2.1.0", default-features = false, features = ["gpu"] } cpu-time = { version = "1.0", optional = true } gperftools = { version = "0.2", optional = true } num_cpus = "1.10.1" -fil-blst = "0.1.1" [dev-dependencies] proptest = "0.10" @@ -54,13 +52,15 @@ pretty_assertions = "0.6.1" sha2raw = { path = "../../sha2raw", version = "^2.0.0"} [features] -default = ["gpu"] +default = ["gpu", "pairing"] simd = [] asm = ["sha2/sha2-asm"] big-sector-sizes-bench = [] gpu = ["bellperson/gpu"] measurements = ["cpu-time", "gperftools"] profile = ["measurements"] +pairing = ["bellperson/pairing", "neptune/pairing", "bellperson/pairing-serde"] +blst = ["bellperson/blst", "neptune/blst", "bellperson/blst-serde"] [[bench]] name = "sha256" diff --git a/storage-proofs/core/benches/blake2s.rs b/storage-proofs/core/benches/blake2s.rs index bca13282e..de14ba8e9 100644 --- a/storage-proofs/core/benches/blake2s.rs +++ b/storage-proofs/core/benches/blake2s.rs @@ -1,9 +1,9 @@ +use bellperson::bls::Bls12; use bellperson::gadgets::boolean::{self, Boolean}; use bellperson::groth16::*; use bellperson::util_cs::bench_cs::BenchCS; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use criterion::{black_box, criterion_group, criterion_main, Criterion, ParameterizedBenchmark}; -use paired::bls12_381::Bls12; use rand::{thread_rng, Rng}; struct Blake2sExample<'a> { diff --git a/storage-proofs/core/benches/fr.rs b/storage-proofs/core/benches/fr.rs index b793be308..0f30f8554 100644 --- a/storage-proofs/core/benches/fr.rs +++ b/storage-proofs/core/benches/fr.rs @@ -1,6 +1,6 @@ +use bellperson::bls::Fr; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use ff::Field; -use paired::bls12_381::Fr; use rand::thread_rng; use storage_proofs_core::fr32::{bytes_into_fr, fr_into_bytes}; diff --git a/storage-proofs/core/benches/sha256.rs b/storage-proofs/core/benches/sha256.rs index 270d68bcf..4b7f10c2e 100644 --- a/storage-proofs/core/benches/sha256.rs +++ b/storage-proofs/core/benches/sha256.rs @@ -1,3 +1,4 @@ +use bellperson::bls::Bls12; use bellperson::gadgets::boolean::{self, Boolean}; use bellperson::groth16::*; use bellperson::util_cs::bench_cs::BenchCS; @@ -5,7 +6,6 @@ use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use criterion::{ black_box, criterion_group, criterion_main, Criterion, ParameterizedBenchmark, Throughput, }; -use paired::bls12_381::Bls12; use rand::{thread_rng, Rng}; use sha2::{Digest, Sha256}; diff --git a/storage-proofs/core/benches/xor.rs b/storage-proofs/core/benches/xor.rs index 71be99751..f3fbd785c 100644 --- a/storage-proofs/core/benches/xor.rs +++ b/storage-proofs/core/benches/xor.rs @@ -1,9 +1,9 @@ +use bellperson::bls::Bls12; use bellperson::gadgets::boolean::{self, Boolean}; use bellperson::groth16::*; use bellperson::util_cs::bench_cs::BenchCS; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use criterion::{black_box, criterion_group, criterion_main, Criterion, ParameterizedBenchmark}; -use paired::bls12_381::Bls12; use rand::{thread_rng, Rng}; use storage_proofs_core::crypto::xor; use storage_proofs_core::gadgets; diff --git a/storage-proofs/core/src/compound_proof.rs b/storage-proofs/core/src/compound_proof.rs index 518531520..a495fded7 100644 --- a/storage-proofs/core/src/compound_proof.rs +++ b/storage-proofs/core/src/compound_proof.rs @@ -1,10 +1,7 @@ -use std::path::Path; - use anyhow::{ensure, Context}; +use bellperson::bls::{Bls12, Fr}; use bellperson::{groth16, Circuit}; -use fil_blst::{blst_fr, blst_scalar, scalar_from_u64, verify_batch_proof}; use log::info; -use paired::bls12_381::{Bls12, Fr}; use rand::{rngs::OsRng, RngCore}; use rayon::prelude::*; @@ -102,7 +99,7 @@ where )?; info!("snark_proof:finish"); - Ok(MultiProof::new(groth_proofs, &groth_params.vk)) + Ok(MultiProof::new(groth_proofs, &groth_params.pvk)) } fn prove_with_vanilla<'b>( @@ -126,7 +123,7 @@ where )?; info!("snark_proof:finish"); - Ok(MultiProof::new(groth_proofs, &groth_params.vk)) + Ok(MultiProof::new(groth_proofs, &groth_params.pvk)) } // verify is equivalent to ProofScheme::verify. @@ -142,7 +139,7 @@ where ); let vanilla_public_params = &public_params.vanilla_params; - let pvk = groth16::prepare_batch_verifying_key(&multi_proof.verifying_key); + let pvk = &multi_proof.verifying_key; if !::satisfies_requirements( &public_params.vanilla_params, @@ -156,72 +153,12 @@ where .into_par_iter() .map(|k| Self::generate_public_inputs(public_inputs, vanilla_public_params, Some(k))) .collect::>()?; - let proofs: Vec<_> = multi_proof.circuit_proofs.iter().collect(); + let proofs: Vec<_> = multi_proof.circuit_proofs.iter().collect(); let res = groth16::verify_proofs_batch(&pvk, &mut rand::rngs::OsRng, &proofs, &inputs)?; Ok(res) } - // verify_blst is equivalent to ProofScheme::verify. - fn verify_blst( - public_params: &PublicParams<'a, S>, - public_inputs: &S::PublicInputs, - proof_vec: &[u8], - num_proofs: usize, - requirements: &S::Requirements, - vk_path: &Path, - ) -> Result { - ensure!( - num_proofs == Self::partition_count(public_params), - "Inconsistent inputs" - ); - - let vanilla_public_params = &public_params.vanilla_params; - - if !::satisfies_requirements( - &public_params.vanilla_params, - requirements, - num_proofs, - ) { - return Ok(false); - } - - let inputs: Vec<_> = (0..num_proofs) - .into_par_iter() - .map(|k| Self::generate_public_inputs(public_inputs, vanilla_public_params, Some(k))) - .collect::>()?; - - let blst_inputs: Vec<_> = inputs - .iter() - .flat_map(|pis| pis.iter().map(|pi| blst_fr::from(*pi))) - .collect(); - - // choose random coefficients for combining the proofs - let mut r: Vec = Vec::with_capacity(num_proofs); - let mut rng = rand::rngs::OsRng; - for _ in 0..num_proofs { - use rand::Rng; - let t: u128 = rng.gen(); - - let mut limbs: [u64; 4] = [0, 0, 0, 0]; - limbs[1] = (t >> 64) as u64; - limbs[0] = (t & (-1i64 as u128) >> 64) as u64; - - r.push(scalar_from_u64(&limbs)); - } - - let res = verify_batch_proof( - proof_vec, - num_proofs, - &blst_inputs, - inputs[0].len(), - &r, - 128, - vk_path, - ); - Ok(res) - } - /// Efficiently verify multiple proofs. fn batch_verify<'b>( public_params: &PublicParams<'a, S>, @@ -243,7 +180,7 @@ where let vanilla_public_params = &public_params.vanilla_params; // just use the first one, the must be equal any way - let pvk = groth16::prepare_batch_verifying_key(&multi_proofs[0].verifying_key); + let pvk = &multi_proofs[0].verifying_key; for multi_proof in multi_proofs.iter() { if !::satisfies_requirements( diff --git a/storage-proofs/core/src/crypto/sloth.rs b/storage-proofs/core/src/crypto/sloth.rs index c4a991aa0..bea2dd2e3 100644 --- a/storage-proofs/core/src/crypto/sloth.rs +++ b/storage-proofs/core/src/crypto/sloth.rs @@ -1,5 +1,5 @@ +use bellperson::bls::Fr; use ff::Field; -use paired::bls12_381::Fr; /// Sloth based encoding. #[inline] @@ -23,8 +23,8 @@ pub fn decode(key: &Fr, ciphertext: &Fr) -> Fr { #[cfg(test)] mod tests { use super::*; + use bellperson::bls::{Fr, FrRepr}; use ff::PrimeField; - use paired::bls12_381::{Fr, FrRepr}; use proptest::{prop_compose, proptest}; // the modulus from `bls12_381::Fr` diff --git a/storage-proofs/core/src/fr32.rs b/storage-proofs/core/src/fr32.rs index 81b64f2e7..a9481eba1 100644 --- a/storage-proofs/core/src/fr32.rs +++ b/storage-proofs/core/src/fr32.rs @@ -1,9 +1,9 @@ use crate::error::*; use anyhow::{ensure, Context}; +use bellperson::bls::{Fr, FrRepr}; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; use ff::{PrimeField, PrimeFieldRepr}; -use paired::bls12_381::{Fr, FrRepr}; // Contains 32 bytes whose little-endian value represents an Fr. // Invariants: @@ -25,6 +25,7 @@ pub type Fr32Ary = [u8; 32]; // Takes a slice of bytes and returns an Fr if byte slice is exactly 32 bytes and does not overflow. // Otherwise, returns a BadFrBytesError. +#[cfg(feature = "pairing")] pub fn bytes_into_fr(bytes: &[u8]) -> Result { ensure!(bytes.len() == 32, Error::BadFrBytes); @@ -34,6 +35,14 @@ pub fn bytes_into_fr(bytes: &[u8]) -> Result { Fr::from_repr(fr_repr).map_err(|_| Error::BadFrBytes.into()) } +#[cfg(feature = "blst")] +pub fn bytes_into_fr(bytes: &[u8]) -> Result { + use std::convert::TryInto; + + Fr::from_bytes_le(bytes.try_into().map_err(|_| Error::BadFrBytes)?) + .ok_or_else(|| Error::BadFrBytes.into()) +} + #[inline] pub fn trim_bytes_to_fr_safe(r: &[u8]) -> Result> { ensure!(r.len() == 32, Error::BadFrBytes); @@ -65,12 +74,20 @@ pub fn bytes_into_fr_repr_safe(r: &[u8]) -> FrRepr { } // Takes an Fr and returns a vector of exactly 32 bytes guaranteed to contain a valid Fr. +#[cfg(feature = "pairing")] pub fn fr_into_bytes(fr: &Fr) -> Fr32Vec { let mut out = Vec::with_capacity(32); fr.into_repr().write_le(&mut out).expect("write_le failure"); out } +#[cfg(feature = "blst")] +pub fn fr_into_bytes(fr: &Fr) -> Fr32Vec { + use std::convert::TryInto; + + fr.to_bytes_le().to_vec() +} + // Takes a slice of bytes and returns a vector of Fr -- or an error if either bytes is not a multiple of 32 bytes // or any 32-byte chunk overflows and does not contain a valid Fr. pub fn bytes_into_frs(bytes: &[u8]) -> Result> { diff --git a/storage-proofs/core/src/gadgets/constraint.rs b/storage-proofs/core/src/gadgets/constraint.rs index d6ae000ed..bffc86f14 100644 --- a/storage-proofs/core/src/gadgets/constraint.rs +++ b/storage-proofs/core/src/gadgets/constraint.rs @@ -1,6 +1,5 @@ -use bellperson::{gadgets::num, ConstraintSystem, SynthesisError}; +use bellperson::{bls::Engine, gadgets::num, ConstraintSystem, SynthesisError}; use ff::Field; -use paired::Engine; /// Adds a constraint to CS, enforcing an equality relationship between the allocated numbers a and b. /// @@ -119,8 +118,8 @@ pub fn difference>( mod tests { use super::*; + use bellperson::bls::{Bls12, Fr}; use bellperson::util_cs::test_cs::TestConstraintSystem; - use paired::bls12_381::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/core/src/gadgets/encode.rs b/storage-proofs/core/src/gadgets/encode.rs index 08486b439..f5d14bd15 100644 --- a/storage-proofs/core/src/gadgets/encode.rs +++ b/storage-proofs/core/src/gadgets/encode.rs @@ -1,6 +1,5 @@ use bellperson::gadgets::num; -use bellperson::{ConstraintSystem, SynthesisError}; -use paired::Engine; +use bellperson::{bls::Engine, ConstraintSystem, SynthesisError}; use crate::gadgets::constraint; diff --git a/storage-proofs/core/src/gadgets/insertion.rs b/storage-proofs/core/src/gadgets/insertion.rs index ca016f954..ad1236cc3 100644 --- a/storage-proofs/core/src/gadgets/insertion.rs +++ b/storage-proofs/core/src/gadgets/insertion.rs @@ -5,9 +5,8 @@ use bellperson::gadgets::boolean::{AllocatedBit, Boolean}; use bellperson::gadgets::num::AllocatedNum; -use bellperson::{ConstraintSystem, SynthesisError}; +use bellperson::{bls::Engine, ConstraintSystem, SynthesisError}; use ff::Field; -use paired::Engine; /// Insert `element` after the nth 1-indexed element of `elements`, where `path_bits` represents n, least-significant bit first. /// The returned result contains a new vector of `AllocatedNum`s with `element` inserted, and constraints are enforced. @@ -349,10 +348,10 @@ where mod tests { use super::*; + use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::boolean::AllocatedBit; use bellperson::util_cs::test_cs::TestConstraintSystem; use ff::Field; - use paired::bls12_381::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/core/src/gadgets/por.rs b/storage-proofs/core/src/gadgets/por.rs index 7566ad0c7..22984a124 100644 --- a/storage-proofs/core/src/gadgets/por.rs +++ b/storage-proofs/core/src/gadgets/por.rs @@ -2,12 +2,12 @@ use std::convert::TryFrom; use std::marker::PhantomData; use anyhow::ensure; +use bellperson::bls::{Bls12, Fr, FrRepr}; use bellperson::gadgets::boolean::{AllocatedBit, Boolean}; use bellperson::gadgets::{multipack, num}; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use ff::PrimeField; use generic_array::typenum::Unsigned; -use paired::bls12_381::{Bls12, Fr, FrRepr}; use crate::compound_proof::{CircuitComponent, CompoundProof}; use crate::error::Result; diff --git a/storage-proofs/core/src/gadgets/uint64.rs b/storage-proofs/core/src/gadgets/uint64.rs index da51f2ae3..34c288ac6 100644 --- a/storage-proofs/core/src/gadgets/uint64.rs +++ b/storage-proofs/core/src/gadgets/uint64.rs @@ -1,7 +1,6 @@ use bellperson::gadgets::boolean::{AllocatedBit, Boolean}; use bellperson::gadgets::multipack::pack_into_inputs; -use bellperson::{ConstraintSystem, SynthesisError}; -use paired::Engine; +use bellperson::{bls::Engine, ConstraintSystem, SynthesisError}; /// Represents an interpretation of 64 `Boolean` objects as an unsigned integer. #[derive(Clone)] diff --git a/storage-proofs/core/src/gadgets/variables.rs b/storage-proofs/core/src/gadgets/variables.rs index 5b6ed606b..2e0e31a97 100644 --- a/storage-proofs/core/src/gadgets/variables.rs +++ b/storage-proofs/core/src/gadgets/variables.rs @@ -3,8 +3,7 @@ use std::fmt; use anyhow::Result; use bellperson::gadgets::num::AllocatedNum; -use bellperson::{ConstraintSystem, SynthesisError}; -use paired::Engine; +use bellperson::{bls::Engine, ConstraintSystem, SynthesisError}; /// Root represents a root commitment which may be either a raw value or an already-allocated number. /// This allows subcomponents to depend on roots which may optionally be shared with their parent diff --git a/storage-proofs/core/src/gadgets/xor.rs b/storage-proofs/core/src/gadgets/xor.rs index eaf127542..157cc9a01 100644 --- a/storage-proofs/core/src/gadgets/xor.rs +++ b/storage-proofs/core/src/gadgets/xor.rs @@ -1,6 +1,5 @@ use bellperson::gadgets::boolean::Boolean; -use bellperson::{ConstraintSystem, SynthesisError}; -use paired::Engine; +use bellperson::{bls::Engine, ConstraintSystem, SynthesisError}; pub fn xor( cs: &mut CS, @@ -35,8 +34,7 @@ mod tests { use crate::util::{bits_to_bytes, bytes_into_boolean_vec}; use bellperson::gadgets::boolean::Boolean; use bellperson::util_cs::test_cs::TestConstraintSystem; - use bellperson::ConstraintSystem; - use paired::bls12_381::Bls12; + use bellperson::{bls::Bls12, ConstraintSystem}; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/core/src/hasher/blake2s.rs b/storage-proofs/core/src/hasher/blake2s.rs index 89df68a81..74e52644c 100644 --- a/storage-proofs/core/src/hasher/blake2s.rs +++ b/storage-proofs/core/src/hasher/blake2s.rs @@ -2,13 +2,13 @@ use std::fmt; use std::hash::Hasher as StdHasher; use anyhow::ensure; +use bellperson::bls::{Bls12, Fr, FrRepr}; use bellperson::gadgets::{blake2s as blake2s_circuit, boolean, num}; use bellperson::{ConstraintSystem, SynthesisError}; use blake2s_simd::{Hash as Blake2sHash, Params as Blake2s, State}; use ff::{Field, PrimeField, PrimeFieldRepr}; use merkletree::hash::{Algorithm, Hashable}; use merkletree::merkle::Element; -use paired::bls12_381::{Bls12, Fr, FrRepr}; use rand::RngCore; use serde::{Deserialize, Serialize}; diff --git a/storage-proofs/core/src/hasher/poseidon.rs b/storage-proofs/core/src/hasher/poseidon.rs index 81e7b8e0d..b71c235a9 100644 --- a/storage-proofs/core/src/hasher/poseidon.rs +++ b/storage-proofs/core/src/hasher/poseidon.rs @@ -8,6 +8,7 @@ use crate::hasher::types::{ }; use crate::hasher::{Domain, HashFunction, Hasher}; use anyhow::ensure; +use bellperson::bls::{Bls12, Fr, FrRepr}; use bellperson::gadgets::{boolean, num}; use bellperson::{ConstraintSystem, SynthesisError}; use ff::{Field, PrimeField, PrimeFieldRepr, ScalarEngine}; @@ -17,7 +18,6 @@ use merkletree::hash::{Algorithm as LightAlgorithm, Hashable}; use merkletree::merkle::Element; use neptune::circuit::poseidon_hash; use neptune::poseidon::Poseidon; -use paired::bls12_381::{Bls12, Fr, FrRepr}; use serde::{Deserialize, Serialize}; #[derive(Default, Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] diff --git a/storage-proofs/core/src/hasher/sha256.rs b/storage-proofs/core/src/hasher/sha256.rs index 96557b0ab..85a36dec7 100644 --- a/storage-proofs/core/src/hasher/sha256.rs +++ b/storage-proofs/core/src/hasher/sha256.rs @@ -1,12 +1,12 @@ use std::hash::Hasher as StdHasher; use anyhow::ensure; +use bellperson::bls::{Bls12, Fr, FrRepr}; use bellperson::gadgets::{boolean, num, sha256::sha256 as sha256_circuit}; use bellperson::{ConstraintSystem, SynthesisError}; use ff::{Field, PrimeField, PrimeFieldRepr}; use merkletree::hash::{Algorithm, Hashable}; use merkletree::merkle::Element; -use paired::bls12_381::{Bls12, Fr, FrRepr}; use rand::RngCore; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -356,11 +356,11 @@ mod tests { use crate::util::bytes_into_boolean_vec; use bellperson::util_cs::test_cs::TestConstraintSystem; + use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::boolean::Boolean; use bellperson::ConstraintSystem; use ff::Field; use merkletree::hash::Algorithm; - use paired::bls12_381::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/core/src/hasher/types.rs b/storage-proofs/core/src/hasher/types.rs index 1e5cfd739..aae9f204c 100644 --- a/storage-proofs/core/src/hasher/types.rs +++ b/storage-proofs/core/src/hasher/types.rs @@ -1,3 +1,4 @@ +use bellperson::bls::{Bls12, Fr, FrRepr}; use bellperson::gadgets::{boolean, num}; use bellperson::{ConstraintSystem, SynthesisError}; use generic_array::typenum::{U0, U11, U16, U2, U24, U36, U4, U8}; @@ -5,7 +6,6 @@ use lazy_static::lazy_static; use merkletree::hash::{Algorithm as LightAlgorithm, Hashable as LightHashable}; use merkletree::merkle::Element; use neptune::poseidon::PoseidonConstants; -use paired::bls12_381::{Bls12, Fr, FrRepr}; use serde::de::DeserializeOwned; use serde::ser::Serialize; diff --git a/storage-proofs/core/src/merkle/proof.rs b/storage-proofs/core/src/merkle/proof.rs index de7278544..5db3cd391 100644 --- a/storage-proofs/core/src/merkle/proof.rs +++ b/storage-proofs/core/src/merkle/proof.rs @@ -3,10 +3,10 @@ use std::marker::PhantomData; use anyhow::{ensure, Result}; +use bellperson::bls::Fr; use generic_array::typenum::{Unsigned, U0}; use merkletree::hash::Algorithm; use merkletree::proof; -use paired::bls12_381::Fr; use serde::{Deserialize, Serialize}; use crate::drgraph::graph_height; diff --git a/storage-proofs/core/src/multi_proof.rs b/storage-proofs/core/src/multi_proof.rs index 0f4360539..350e59531 100644 --- a/storage-proofs/core/src/multi_proof.rs +++ b/storage-proofs/core/src/multi_proof.rs @@ -2,13 +2,12 @@ use bellperson::groth16; use crate::error::Result; use anyhow::{ensure, Context}; -use paired::bls12_381::Bls12; -use rayon::prelude::*; -use std::io::{self, Read, Write}; +use bellperson::bls::Bls12; +use std::io::{Read, Write}; pub struct MultiProof<'a> { pub circuit_proofs: Vec>, - pub verifying_key: &'a groth16::VerifyingKey, + pub verifying_key: &'a groth16::PreparedVerifyingKey, } const GROTH_PROOF_SIZE: usize = 192; @@ -16,7 +15,7 @@ const GROTH_PROOF_SIZE: usize = 192; impl<'a> MultiProof<'a> { pub fn new( groth_proofs: Vec>, - verifying_key: &'a groth16::VerifyingKey, + verifying_key: &'a groth16::PreparedVerifyingKey, ) -> Self { MultiProof { circuit_proofs: groth_proofs, @@ -27,7 +26,7 @@ impl<'a> MultiProof<'a> { pub fn new_from_reader( partitions: Option, mut reader: R, - verifying_key: &'a groth16::VerifyingKey, + verifying_key: &'a groth16::PreparedVerifyingKey, ) -> Result { let num_proofs = partitions.unwrap_or(1); @@ -41,15 +40,11 @@ impl<'a> MultiProof<'a> { pub fn new_from_bytes( partitions: Option, proof_bytes: &[u8], - verifying_key: &'a groth16::VerifyingKey, + verifying_key: &'a groth16::PreparedVerifyingKey, ) -> Result { let num_proofs = partitions.unwrap_or(1); - let proofs = proof_bytes - .par_chunks(GROTH_PROOF_SIZE) - .take(num_proofs) - .map(groth16::Proof::read) - .collect::>>()?; + let proofs = groth16::Proof::read_many(proof_bytes, num_proofs)?; ensure!( num_proofs == proofs.len(), diff --git a/storage-proofs/core/src/parameter_cache.rs b/storage-proofs/core/src/parameter_cache.rs index b1e34864a..603ef1a11 100644 --- a/storage-proofs/core/src/parameter_cache.rs +++ b/storage-proofs/core/src/parameter_cache.rs @@ -1,5 +1,6 @@ use crate::error::*; use anyhow::bail; +use bellperson::bls::Bls12; use bellperson::groth16::Parameters; use bellperson::{groth16, Circuit}; use blake2b_simd::Params as Blake2bParams; @@ -7,7 +8,6 @@ use fs2::FileExt; use itertools::Itertools; use lazy_static::lazy_static; use log::info; -use paired::bls12_381::Bls12; use rand::RngCore; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; diff --git a/storage-proofs/core/src/por.rs b/storage-proofs/core/src/por.rs index c2e60edfd..a96f1227a 100644 --- a/storage-proofs/core/src/por.rs +++ b/storage-proofs/core/src/por.rs @@ -150,9 +150,9 @@ impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for PoR { mod tests { use super::*; + use bellperson::bls::Fr; use ff::Field; use generic_array::typenum; - use paired::bls12_381::Fr; use rand::SeedableRng; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/core/src/sector.rs b/storage-proofs/core/src/sector.rs index 26f63def1..223214292 100644 --- a/storage-proofs/core/src/sector.rs +++ b/storage-proofs/core/src/sector.rs @@ -1,9 +1,9 @@ use std::collections::BTreeSet; use std::fmt; +use bellperson::bls::{Fr, FrRepr}; use byteorder::ByteOrder; use ff::PrimeField; -use paired::bls12_381::{Fr, FrRepr}; use serde::{Deserialize, Serialize}; /// An ordered set of `SectorId`s. diff --git a/storage-proofs/core/src/settings.rs b/storage-proofs/core/src/settings.rs index 39e7bb5a5..eb0786243 100644 --- a/storage-proofs/core/src/settings.rs +++ b/storage-proofs/core/src/settings.rs @@ -26,7 +26,6 @@ pub struct Settings { pub window_post_synthesis_num_cpus: u32, pub parameter_cache: String, pub parent_cache: String, - pub use_fil_blst: bool, pub use_multicore_sdr: bool, pub multicore_sdr_producers: usize, pub multicore_sdr_producer_stride: u64, @@ -51,7 +50,6 @@ impl Default for Settings { // The name is retained for backwards compatibility. parameter_cache: "/var/tmp/filecoin-proof-parameters/".to_string(), parent_cache: cache("filecoin-parents"), - use_fil_blst: false, use_multicore_sdr: false, multicore_sdr_producers: 3, multicore_sdr_producer_stride: 128, diff --git a/storage-proofs/core/src/util.rs b/storage-proofs/core/src/util.rs index 94cabac00..5940d2470 100644 --- a/storage-proofs/core/src/util.rs +++ b/storage-proofs/core/src/util.rs @@ -1,9 +1,8 @@ use crate::error; use anyhow::ensure; use bellperson::gadgets::boolean::{self, AllocatedBit, Boolean}; -use bellperson::{ConstraintSystem, SynthesisError}; +use bellperson::{bls::Engine, ConstraintSystem, SynthesisError}; use merkletree::merkle::get_merkle_tree_row_count; -use paired::Engine; use super::settings; @@ -180,10 +179,10 @@ mod tests { use super::*; use crate::fr32::fr_into_bytes; + use bellperson::bls::*; use bellperson::gadgets::num; use bellperson::util_cs::test_cs::TestConstraintSystem; use ff::Field; - use paired::bls12_381::*; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/porep/Cargo.toml b/storage-proofs/porep/Cargo.toml index 425e76ba3..d66a6a289 100644 --- a/storage-proofs/porep/Cargo.toml +++ b/storage-proofs/porep/Cargo.toml @@ -9,9 +9,9 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs" readme = "README.md" [dependencies] -crossbeam = "0.7.3" +crossbeam = "0.8" digest = "0.9" -storage-proofs-core = { path = "../core", version = "^5.0.0"} +storage-proofs-core = { path = "../core", version = "^5.0.0", default-features = false} sha2raw = { path = "../../sha2raw", version = "^2.0.0"} rand = "0.7" merkletree = "0.21.0" @@ -23,19 +23,18 @@ rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" ff = { version = "0.2.3", package = "fff" } -bellperson = "0.9.1" -paired = { version = "0.20.0", features = ["serde"] } +bellperson = { version = "0.11", default-features = false } log = "0.4.7" pretty_assertions = "0.6.1" -generic-array = "0.13.2" +generic-array = "0.14.4" anyhow = "1.0.23" -neptune = { version = "=1.2.1", features = ["gpu"] } +neptune = { version = "2.1.0", default-features = false, features = ["gpu"] } num_cpus = "1.10.1" hex = "0.4.2" bincode = "1.1.2" byteorder = "1.3.4" lazy_static = "1.2" -byte-slice-cast = "0.3.5" +byte-slice-cast = "1.0.0" hwloc = "0.3.0" libc = "0.2" @@ -47,7 +46,10 @@ glob = "0.3.0" pretty_env_logger = "0.4.0" [features] -default = [] +default = ["pairing", "gpu"] +gpu = ["storage-proofs-core/gpu"] +pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing"] +blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst"] [[bench]] name = "encode" diff --git a/storage-proofs/porep/benches/encode.rs b/storage-proofs/porep/benches/encode.rs index dde844a5d..40c723805 100644 --- a/storage-proofs/porep/benches/encode.rs +++ b/storage-proofs/porep/benches/encode.rs @@ -1,6 +1,6 @@ +use bellperson::bls::Fr; use criterion::{black_box, criterion_group, criterion_main, Criterion, Throughput}; use ff::Field; -use paired::bls12_381::Fr; use rand::thread_rng; use storage_proofs_core::fr32::fr_into_bytes; use storage_proofs_core::hasher::sha256::Sha256Hasher; diff --git a/storage-proofs/porep/src/drg/circuit.rs b/storage-proofs/porep/src/drg/circuit.rs index bbc19dab3..a77fe9578 100644 --- a/storage-proofs/porep/src/drg/circuit.rs +++ b/storage-proofs/porep/src/drg/circuit.rs @@ -1,5 +1,6 @@ use std::marker::PhantomData; +use bellperson::bls::{Bls12, Engine, Fr}; use bellperson::gadgets::{ boolean::Boolean, sha256::sha256 as sha256_circuit, @@ -7,8 +8,6 @@ use bellperson::gadgets::{ }; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use ff::PrimeField; -use paired::bls12_381::{Bls12, Fr}; -use paired::Engine; use storage_proofs_core::{ compound_proof::CircuitComponent, error::Result, gadgets::constraint, gadgets::encode, diff --git a/storage-proofs/porep/src/drg/compound.rs b/storage-proofs/porep/src/drg/compound.rs index e9cf4ff89..1eead8b19 100644 --- a/storage-proofs/porep/src/drg/compound.rs +++ b/storage-proofs/porep/src/drg/compound.rs @@ -1,9 +1,9 @@ use std::marker::PhantomData; use anyhow::{ensure, Context}; +use bellperson::bls::{Bls12, Fr}; use bellperson::Circuit; use generic_array::typenum; -use paired::bls12_381::{Bls12, Fr}; use storage_proofs_core::{ compound_proof::{CircuitComponent, CompoundProof}, diff --git a/storage-proofs/porep/src/drg/vanilla.rs b/storage-proofs/porep/src/drg/vanilla.rs index 4d9f474d6..f89515974 100644 --- a/storage-proofs/porep/src/drg/vanilla.rs +++ b/storage-proofs/porep/src/drg/vanilla.rs @@ -607,8 +607,8 @@ pub fn replica_id(prover_id: [u8; 32], sector_id: [u8; 32]) -> H::Dom mod tests { use super::*; + use bellperson::bls::Fr; use ff::Field; - use paired::bls12_381::Fr; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ diff --git a/storage-proofs/porep/src/encode.rs b/storage-proofs/porep/src/encode.rs index ca3733e28..081983041 100644 --- a/storage-proofs/porep/src/encode.rs +++ b/storage-proofs/porep/src/encode.rs @@ -1,5 +1,5 @@ +use bellperson::bls::Fr; use ff::Field; -use paired::bls12_381::Fr; use storage_proofs_core::hasher::Domain; pub fn encode(key: T, value: T) -> T { diff --git a/storage-proofs/porep/src/stacked/circuit/column.rs b/storage-proofs/porep/src/stacked/circuit/column.rs index 6b5dabe0a..417bab73f 100644 --- a/storage-proofs/porep/src/stacked/circuit/column.rs +++ b/storage-proofs/porep/src/stacked/circuit/column.rs @@ -1,6 +1,6 @@ +use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::num; use bellperson::{ConstraintSystem, SynthesisError}; -use paired::bls12_381::{Bls12, Fr}; use storage_proofs_core::{hasher::Hasher, merkle::MerkleTreeTrait}; use super::hash::hash_single_column; diff --git a/storage-proofs/porep/src/stacked/circuit/column_proof.rs b/storage-proofs/porep/src/stacked/circuit/column_proof.rs index dc6c255df..258cef89e 100644 --- a/storage-proofs/porep/src/stacked/circuit/column_proof.rs +++ b/storage-proofs/porep/src/stacked/circuit/column_proof.rs @@ -1,5 +1,5 @@ +use bellperson::bls::Bls12; use bellperson::{ConstraintSystem, SynthesisError}; -use paired::bls12_381::Bls12; use storage_proofs_core::{ drgraph::Graph, gadgets::por::AuthPath, diff --git a/storage-proofs/porep/src/stacked/circuit/create_label.rs b/storage-proofs/porep/src/stacked/circuit/create_label.rs index e56206421..850dfacc4 100644 --- a/storage-proofs/porep/src/stacked/circuit/create_label.rs +++ b/storage-proofs/porep/src/stacked/circuit/create_label.rs @@ -1,7 +1,6 @@ use bellperson::gadgets::{boolean::Boolean, num, sha256::sha256 as sha256_circuit, uint32}; -use bellperson::{ConstraintSystem, SynthesisError}; +use bellperson::{bls::Engine, ConstraintSystem, SynthesisError}; use ff::PrimeField; -use paired::Engine; use storage_proofs_core::{gadgets::multipack, gadgets::uint64, util::reverse_bit_numbering}; use crate::stacked::vanilla::TOTAL_PARENTS; @@ -68,10 +67,10 @@ where mod tests { use super::*; + use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::boolean::Boolean; use bellperson::util_cs::test_cs::TestConstraintSystem; use ff::Field; - use paired::bls12_381::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ diff --git a/storage-proofs/porep/src/stacked/circuit/hash.rs b/storage-proofs/porep/src/stacked/circuit/hash.rs index cd0069333..20e4ff626 100644 --- a/storage-proofs/porep/src/stacked/circuit/hash.rs +++ b/storage-proofs/porep/src/stacked/circuit/hash.rs @@ -1,8 +1,8 @@ +use bellperson::bls::Bls12; use bellperson::gadgets::num; use bellperson::{ConstraintSystem, SynthesisError}; use generic_array::typenum; use neptune::circuit::poseidon_hash; -use paired::bls12_381::Bls12; /// Hash a list of bits. pub fn hash_single_column( @@ -31,10 +31,10 @@ where mod tests { use super::*; + use bellperson::bls::{Bls12, Fr}; use bellperson::util_cs::test_cs::TestConstraintSystem; use bellperson::ConstraintSystem; use ff::Field; - use paired::bls12_381::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::hasher::{HashFunction, Hasher, PoseidonHasher}; diff --git a/storage-proofs/porep/src/stacked/circuit/params.rs b/storage-proofs/porep/src/stacked/circuit/params.rs index bd5f5de65..985bad3d1 100644 --- a/storage-proofs/porep/src/stacked/circuit/params.rs +++ b/storage-proofs/porep/src/stacked/circuit/params.rs @@ -1,9 +1,9 @@ use std::marker::PhantomData; +use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::{boolean::Boolean, num, uint32}; use bellperson::{ConstraintSystem, SynthesisError}; use generic_array::typenum::{U0, U2}; -use paired::bls12_381::{Bls12, Fr}; use storage_proofs_core::{ drgraph::Graph, gadgets::por::{AuthPath, PoRCircuit}, diff --git a/storage-proofs/porep/src/stacked/circuit/proof.rs b/storage-proofs/porep/src/stacked/circuit/proof.rs index aed1549d8..d9fbe4d5e 100644 --- a/storage-proofs/porep/src/stacked/circuit/proof.rs +++ b/storage-proofs/porep/src/stacked/circuit/proof.rs @@ -1,9 +1,9 @@ use std::marker::PhantomData; use anyhow::ensure; +use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::num; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; -use paired::bls12_381::{Bls12, Fr}; use storage_proofs_core::{ compound_proof::{CircuitComponent, CompoundProof}, drgraph::Graph, diff --git a/storage-proofs/porep/src/stacked/vanilla/column.rs b/storage-proofs/porep/src/stacked/vanilla/column.rs index efdb1c545..73fb50b87 100644 --- a/storage-proofs/porep/src/stacked/vanilla/column.rs +++ b/storage-proofs/porep/src/stacked/vanilla/column.rs @@ -1,6 +1,6 @@ use std::marker::PhantomData; -use paired::bls12_381::Fr; +use bellperson::bls::Fr; use serde::{Deserialize, Serialize}; use storage_proofs_core::{ error::Result, diff --git a/storage-proofs/porep/src/stacked/vanilla/column_proof.rs b/storage-proofs/porep/src/stacked/vanilla/column_proof.rs index 17632a0b3..137da4d30 100644 --- a/storage-proofs/porep/src/stacked/vanilla/column_proof.rs +++ b/storage-proofs/porep/src/stacked/vanilla/column_proof.rs @@ -1,5 +1,5 @@ +use bellperson::bls::Fr; use log::trace; -use paired::bls12_381::Fr; use serde::{Deserialize, Serialize}; use storage_proofs_core::{error::Result, hasher::Hasher, merkle::MerkleProofTrait}; diff --git a/storage-proofs/porep/src/stacked/vanilla/create_label/multi.rs b/storage-proofs/porep/src/stacked/vanilla/create_label/multi.rs index 17fa51ec5..36c1d8feb 100644 --- a/storage-proofs/porep/src/stacked/vanilla/create_label/multi.rs +++ b/storage-proofs/porep/src/stacked/vanilla/create_label/multi.rs @@ -619,9 +619,9 @@ pub fn create_labels_for_decoding StackedDrg<'a, Tr mod tests { use super::*; + use bellperson::bls::{Fr, FrRepr}; use ff::{Field, PrimeField}; - use paired::bls12_381::{Fr, FrRepr}; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; use storage_proofs_core::hasher::poseidon::PoseidonHasher; diff --git a/storage-proofs/post/Cargo.toml b/storage-proofs/post/Cargo.toml index 334f2b4e1..bc35dc9f7 100644 --- a/storage-proofs/post/Cargo.toml +++ b/storage-proofs/post/Cargo.toml @@ -9,27 +9,33 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs" readme = "README.md" [dependencies] -storage-proofs-core = { path = "../core", version = "^5.0.0"} +storage-proofs-core = { path = "../core", version = "^5.0.0", default-features = false} rand = "0.7" merkletree = "0.21.0" byteorder = "1" -crossbeam = "0.7.3" +crossbeam = "0.8" sha2 = "0.9.1" rayon = "1.0.0" serde = { version = "1.0", features = ["derive"]} blake2b_simd = "0.5" blake2s_simd = "0.5" ff = { version = "0.2.3", package = "fff" } -bellperson = "0.9.1" -paired = { version = "0.20.0", features = ["serde"] } +bellperson = { version = "0.11", default-features = false } log = "0.4.7" hex = "0.4.0" -generic-array = "0.13.2" +generic-array = "0.14.4" anyhow = "1.0.23" -neptune = { version = "=1.2.1", features = ["gpu"] } +neptune = { version = "2.1.0", default-features = false, features = ["gpu"] } num_cpus = "1.10.1" [dev-dependencies] tempfile = "3" pretty_assertions = "0.6.1" rand_xorshift = "0.2.0" + +[features] +default = ["pairing", "gpu"] +gpu = ["storage-proofs-core/gpu"] +pairing = ["storage-proofs-core/pairing", "bellperson/pairing", "neptune/pairing"] +blst = ["storage-proofs-core/blst", "bellperson/blst", "neptune/blst"] + diff --git a/storage-proofs/post/src/election/circuit.rs b/storage-proofs/post/src/election/circuit.rs index ead924fa5..7a8dcee71 100644 --- a/storage-proofs/post/src/election/circuit.rs +++ b/storage-proofs/post/src/election/circuit.rs @@ -1,10 +1,10 @@ use std::marker::PhantomData; +use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::num; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use ff::Field; use generic_array::typenum; -use paired::bls12_381::{Bls12, Fr}; use typenum::marker_traits::Unsigned; use storage_proofs_core::{ @@ -178,9 +178,9 @@ mod tests { use std::collections::BTreeMap; + use bellperson::bls::{Bls12, Fr}; use bellperson::util_cs::test_cs::TestConstraintSystem; use ff::Field; - use paired::bls12_381::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ diff --git a/storage-proofs/post/src/election/compound.rs b/storage-proofs/post/src/election/compound.rs index 31add334c..9171b3b43 100644 --- a/storage-proofs/post/src/election/compound.rs +++ b/storage-proofs/post/src/election/compound.rs @@ -1,8 +1,8 @@ use std::marker::PhantomData; +use bellperson::bls::{Bls12, Fr}; use bellperson::Circuit; use generic_array::typenum; -use paired::bls12_381::{Bls12, Fr}; use typenum::marker_traits::Unsigned; use storage_proofs_core::{ diff --git a/storage-proofs/post/src/election/vanilla.rs b/storage-proofs/post/src/election/vanilla.rs index 0a2198651..f75237d38 100644 --- a/storage-proofs/post/src/election/vanilla.rs +++ b/storage-proofs/post/src/election/vanilla.rs @@ -3,10 +3,10 @@ use std::fmt; use std::marker::PhantomData; use anyhow::{bail, ensure, Context}; +use bellperson::bls::Fr; use byteorder::{ByteOrder, LittleEndian}; use generic_array::typenum; use log::trace; -use paired::bls12_381::Fr; use rayon::prelude::*; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; diff --git a/storage-proofs/post/src/fallback/circuit.rs b/storage-proofs/post/src/fallback/circuit.rs index 6ffd8932d..5068096ac 100644 --- a/storage-proofs/post/src/fallback/circuit.rs +++ b/storage-proofs/post/src/fallback/circuit.rs @@ -1,7 +1,7 @@ +use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::num; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use ff::Field; -use paired::bls12_381::{Bls12, Fr}; use rayon::prelude::*; use storage_proofs_core::{ @@ -218,7 +218,6 @@ mod tests { use bellperson::util_cs::test_cs::TestConstraintSystem; use ff::Field; use generic_array::typenum::{U0, U2, U4, U8}; - use paired::bls12_381::{Bls12, Fr}; use rand::SeedableRng; use rand_xorshift::XorShiftRng; use storage_proofs_core::{ diff --git a/storage-proofs/post/src/fallback/compound.rs b/storage-proofs/post/src/fallback/compound.rs index b74bd15a5..702318b79 100644 --- a/storage-proofs/post/src/fallback/compound.rs +++ b/storage-proofs/post/src/fallback/compound.rs @@ -1,8 +1,8 @@ use std::marker::PhantomData; use anyhow::{anyhow, ensure}; +use bellperson::bls::{Bls12, Fr}; use bellperson::Circuit; -use paired::bls12_381::{Bls12, Fr}; use storage_proofs_core::{ compound_proof::{CircuitComponent, CompoundProof}, diff --git a/storage-proofs/post/src/fallback/vanilla.rs b/storage-proofs/post/src/fallback/vanilla.rs index 9dcee21f0..473be1dd6 100644 --- a/storage-proofs/post/src/fallback/vanilla.rs +++ b/storage-proofs/post/src/fallback/vanilla.rs @@ -2,10 +2,10 @@ use std::collections::BTreeSet; use std::marker::PhantomData; use anyhow::ensure; +use bellperson::bls::Fr; use byteorder::{ByteOrder, LittleEndian}; use generic_array::typenum::Unsigned; use log::{error, trace}; -use paired::bls12_381::Fr; use rayon::prelude::*; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -205,11 +205,14 @@ pub fn generate_leaf_challenges( ) -> Vec { let mut challenges = Vec::with_capacity(challenge_count); + let mut hasher = Sha256::new(); + hasher.update(AsRef::<[u8]>::as_ref(&randomness)); + hasher.update(§or_id.to_le_bytes()[..]); + for leaf_challenge_index in 0..challenge_count { - let challenge = generate_leaf_challenge( + let challenge = generate_leaf_challenge_inner::( + hasher.clone(), pub_params, - randomness, - sector_id, leaf_challenge_index as u64, ); challenges.push(challenge) @@ -228,6 +231,15 @@ pub fn generate_leaf_challenge( let mut hasher = Sha256::new(); hasher.update(AsRef::<[u8]>::as_ref(&randomness)); hasher.update(§or_id.to_le_bytes()[..]); + + generate_leaf_challenge_inner::(hasher, pub_params, leaf_challenge_index) +} + +fn generate_leaf_challenge_inner( + mut hasher: Sha256, + pub_params: &PublicParams, + leaf_challenge_index: u64, +) -> u64 { hasher.update(&leaf_challenge_index.to_le_bytes()[..]); let hash = hasher.finalize(); @@ -383,6 +395,11 @@ impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for FallbackPoSt<'a, Tree> Tree::Arity::to_usize(), ); + // avoid rehashing fixed inputs + let mut challenge_hasher = Sha256::new(); + challenge_hasher.update(AsRef::<[u8]>::as_ref(&pub_inputs.randomness)); + challenge_hasher.update(&u64::from(sector_id).to_le_bytes()[..]); + let mut inclusion_proofs = Vec::new(); for proof_or_fault in (0..pub_params.challenge_count) .into_par_iter() @@ -390,12 +407,12 @@ impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for FallbackPoSt<'a, Tree> let challenge_index = ((j * num_sectors_per_chunk + i) * pub_params.challenge_count + n) as u64; - let challenged_leaf_start = generate_leaf_challenge( - pub_params, - pub_inputs.randomness, - sector_id.into(), - challenge_index, - ); + let challenged_leaf_start = + generate_leaf_challenge_inner::<::Domain>( + challenge_hasher.clone(), + pub_params, + challenge_index, + ); let proof = tree.gen_cached_proof( challenged_leaf_start as usize, @@ -522,35 +539,51 @@ impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for FallbackPoSt<'a, Tree> inclusion_proofs.len() ); - for (n, inclusion_proof) in inclusion_proofs.iter().enumerate() { - let challenge_index = - ((j * num_sectors_per_chunk + i) * pub_params.challenge_count + n) as u64; - let challenged_leaf_start = generate_leaf_challenge( - pub_params, - pub_inputs.randomness, - sector_id.into(), - challenge_index, - ); - - // validate all comm_r_lasts match - if inclusion_proof.root() != comm_r_last { - error!("inclusion proof root != comm_r_last: {:?}", sector_id); - return Ok(false); - } + // avoid rehashing fixed inputs + let mut challenge_hasher = Sha256::new(); + challenge_hasher.update(AsRef::<[u8]>::as_ref(&pub_inputs.randomness)); + challenge_hasher.update(&u64::from(sector_id).to_le_bytes()[..]); - // validate the path length - let expected_path_length = - inclusion_proof.expected_len(pub_params.sector_size as usize / NODE_SIZE); + let is_valid_list = inclusion_proofs + .par_iter() + .enumerate() + .map(|(n, inclusion_proof)| -> Result { + let challenge_index = ((j * num_sectors_per_chunk + i) + * pub_params.challenge_count + + n) as u64; + let challenged_leaf_start = + generate_leaf_challenge_inner::<::Domain>( + challenge_hasher.clone(), + pub_params, + challenge_index, + ); + + // validate all comm_r_lasts match + if inclusion_proof.root() != comm_r_last { + error!("inclusion proof root != comm_r_last: {:?}", sector_id); + return Ok(false); + } - if expected_path_length != inclusion_proof.path().len() { - error!("wrong path length: {:?}", sector_id); - return Ok(false); - } + // validate the path length + let expected_path_length = inclusion_proof + .expected_len(pub_params.sector_size as usize / NODE_SIZE); - if !inclusion_proof.validate(challenged_leaf_start as usize) { - error!("invalid inclusion proof: {:?}", sector_id); - return Ok(false); - } + if expected_path_length != inclusion_proof.path().len() { + error!("wrong path length: {:?}", sector_id); + return Ok(false); + } + + if !inclusion_proof.validate(challenged_leaf_start as usize) { + error!("invalid inclusion proof: {:?}", sector_id); + return Ok(false); + } + Ok(true) + }) + .collect::>>()?; + + let is_valid = is_valid_list.into_iter().all(|v| v); + if !is_valid { + return Ok(false); } } } diff --git a/storage-proofs/post/src/rational/circuit.rs b/storage-proofs/post/src/rational/circuit.rs index adadc356f..d2b25df4e 100644 --- a/storage-proofs/post/src/rational/circuit.rs +++ b/storage-proofs/post/src/rational/circuit.rs @@ -1,8 +1,8 @@ use std::marker::PhantomData; +use bellperson::bls::{Bls12, Fr}; use bellperson::gadgets::num; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; -use paired::bls12_381::{Bls12, Fr}; use storage_proofs_core::{ compound_proof::CircuitComponent, @@ -111,8 +111,8 @@ mod tests { use std::collections::BTreeMap; + use bellperson::bls::{Bls12, Fr}; use ff::Field; - use paired::bls12_381::{Bls12, Fr}; use rand::{Rng, SeedableRng}; use rand_xorshift::XorShiftRng; diff --git a/storage-proofs/post/src/rational/compound.rs b/storage-proofs/post/src/rational/compound.rs index 891566052..e89f4ce8f 100644 --- a/storage-proofs/post/src/rational/compound.rs +++ b/storage-proofs/post/src/rational/compound.rs @@ -1,9 +1,9 @@ use std::marker::PhantomData; use anyhow::ensure; +use bellperson::bls::{Bls12, Fr}; use bellperson::{Circuit, ConstraintSystem, SynthesisError}; use generic_array::typenum; -use paired::bls12_381::{Bls12, Fr}; use storage_proofs_core::{ compound_proof::{CircuitComponent, CompoundProof},