[WIP] feat: add optional blst backend and verification optimizations

.circleci/config.yml

@@ -29,6 +29,7 @@ jobs:
             - cargo-v28-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
+      - run: rustup install nightly
       - run: rustup component add rustfmt-preview
       - run: rustup component add clippy
       - run: cargo update
@@ -127,9 +128,10 @@ jobs:
             RUST_TEST_THREADS: 1
           no_output_timeout: 30m
 
-  # Running with `use_fil_blst=true` should be integrated directly into the test code. For now we
-  # just re-run the tests that exercise the fil-blst code path with that setting set.
-  test_fil_blst:
+
+  # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we
+  # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set.
+  test_multicore_sdr:
     docker:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
@@ -144,19 +146,28 @@ jobs:
             - cargo-v28-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - restore_parameter_cache
       - run:
-          name: Test with fil-blst enabled
+          name: Test with use_multicore_sdr pairing enabled
           command: |
             ulimit -n 20000
             ulimit -u 20000
             ulimit -n 20000
-            cargo +$(cat rust-toolchain) test --verbose --release --test api -- --ignored
+            cargo +nightly -Zpackage-features test --all --verbose --release --test api -- --ignored lifecycle
           environment:
             RUST_TEST_THREADS: 1
-            FIL_PROOFS_USE_FIL_BLST: true
+            FIL_PROOFS_USE_MULTICORE_SDR: true
 
-  # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we
-  # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set.
-  test_multicore_sdr:
+      - run:
+          name: Test with use_multicore_sdr and blst enabled
+          command: |
+            ulimit -n 20000
+            ulimit -u 20000
+            ulimit -n 20000
+            cargo +nightly -Zpackage-features test --all --no-default-features --features gpu,blst --verbose --release --test api -- --ignored lifecycle
+          environment:
+            RUST_TEST_THREADS: 1
+            FIL_PROOFS_USE_MULTICORE_SDR: true
+
+  test_blst:
     docker:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
@@ -171,15 +182,23 @@ jobs:
             - cargo-v28-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - restore_parameter_cache
       - run:
-          name: Test with use_multicore_sdr enabled
+          name: Test ignored with blst enabled 
           command: |
             ulimit -n 20000
             ulimit -u 20000
             ulimit -n 20000
-            cargo +$(cat rust-toolchain) test --verbose --release -- --ignored lifecycle
+            cargo +nightly -Zpackage-features test --all --no-default-features --features gpu,blst --verbose --release --test api -- --ignored
           environment:
             RUST_TEST_THREADS: 1
-            FIL_PROOFS_USE_MULTICORE_SDR: true
+
+      - run:
+          name: Test with blst enabled 
+          command: |
+            ulimit -n 20000
+            ulimit -u 20000
+            ulimit -n 20000
+            cargo +nightly -Zpackage-features test --all --no-default-features --features gpu,blst --verbose
+
 
   bench:
     docker:
@@ -415,11 +434,12 @@ workflows:
           requires:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux
-      - test_fil_blst:
+
+      - test_multicore_sdr:
           requires:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux
-      - test_multicore_sdr:
+      - test_blst:
           requires:
             - cargo_fetch
             - ensure_groth_parameters_and_keys_linux

fil-proofs-tooling/Cargo.toml

@@ -10,6 +10,8 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs"
 readme = "README.md"
 
 [dependencies]
+storage-proofs = { path = "../storage-proofs", default-features = false }
+filecoin-proofs = { path = "../filecoin-proofs", default-features = false }
 clap = "2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
@@ -20,11 +22,8 @@ regex = "1.3.7"
 commandspec = "0.12.2"
 chrono = { version = "0.4.7", features = ["serde"] }
 memmap = "0.7.0"
-bellperson = "0.9.1"
-paired = "0.20.0"
+bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false }
 rand = "0.7"
-storage-proofs = { path = "../storage-proofs"}
-filecoin-proofs = { path = "../filecoin-proofs"}
 tempfile = "3.0.8"
 cpu-time = "1.0.0"
 git2 = "0.13.6"
@@ -41,17 +40,18 @@ ff = { version = "0.2.3", package = "fff" }
 rand_xorshift = "0.2.0"
 bytefmt = "0.1.7"
 rayon = "1.3.0"
-flexi_logger = "0.14.7"
+flexi_logger = "0.16.1"
 typenum = "1.11.2"
-generic-array = "0.13.2"
+generic-array = "0.14.4"
 byte-unit = "4.0.9"
 
 [features]
 default = ["gpu", "measurements"]
 gpu = ["storage-proofs/gpu", "filecoin-proofs/gpu", "bellperson/gpu"]
 measurements = ["storage-proofs/measurements"]
 profile = ["storage-proofs/profile", "measurements"]
-
+pairing = ["storage-proofs/pairing", "filecoin-proofs/pairing", "bellperson/pairing"]
+blst = ["storage-proofs/blst", "filecoin-proofs/blst", "bellperson/blst"]
 
 [target.'cfg(target_arch = "x86_64")'.dependencies]
-raw-cpuid = "7.0.3"
+raw-cpuid = "8.1.2"

fil-proofs-tooling/src/bin/benchy/hash_fns.rs

@@ -1,8 +1,8 @@
+use bellperson::bls::Bls12;
 use bellperson::gadgets::boolean::Boolean;
 use bellperson::util_cs::test_cs::TestConstraintSystem;
 use bellperson::ConstraintSystem;
 use fil_proofs_tooling::metadata::Metadata;
-use paired::bls12_381::Bls12;
 use rand::RngCore;
 use serde::Serialize;
 use storage_proofs::util::{bits_to_bytes, bytes_into_boolean_vec, bytes_into_boolean_vec_be};

fil-proofs-tooling/src/bin/benchy/prodbench.rs

@@ -1,3 +1,4 @@
+use bellperson::bls::Bls12;
 use bellperson::util_cs::bench_cs::BenchCS;
 use bellperson::Circuit;
 use fil_proofs_tooling::shared::{create_replicas, PROVER_ID, RANDOMNESS, TICKET_BYTES};
@@ -11,7 +12,6 @@ use filecoin_proofs::{
     validate_cache_for_commit, PoRepConfig,
 };
 use log::info;
-use paired::bls12_381::Bls12;
 use rand::SeedableRng;
 use rand_xorshift::XorShiftRng;
 use serde::{Deserialize, Serialize};

fil-proofs-tooling/src/bin/check_parameters/main.rs

@@ -1,9 +1,9 @@
 use std::path::PathBuf;
 
 use anyhow::Result;
+use bellperson::bls::Bls12;
 use bellperson::groth16::MappedParameters;
 use clap::{value_t, App, Arg, SubCommand};
-use paired::bls12_381::Bls12;
 
 use storage_proofs::parameter_cache::read_cached_params;
 

filecoin-proofs/Cargo.toml

@@ -9,7 +9,7 @@ repository = "https://github.com/filecoin-project/rust-fil-proofs"
 readme = "README.md"
 
 [dependencies]
-storage-proofs = { version = "^5.0.0", path = "../storage-proofs" }
+storage-proofs = { version = "^5.0.0", path = "../storage-proofs", default-features = false }
 bitvec = "0.17"
 chrono = "0.4"
 rand = "0.7"
@@ -23,8 +23,7 @@ serde_json = "1.0"
 regex = "1.3.7"
 ff = { version = "0.2.3", package = "fff" }
 blake2b_simd = "0.5"
-bellperson = "0.9.1"
-paired = "0.20.0"
+bellperson = { git = "https://github.com/filecoin-project/bellperson", branch = "blstrs", default-features = false }
 clap = "2"
 log = "0.4.7"
 fil_logger = "0.1"
@@ -42,14 +41,14 @@ sha2 = "0.9.1"
 typenum = "1.11.2"
 bitintr = "0.3.0"
 gperftools = { version = "0.2", optional = true }
-phase2 = { version = "0.8.0", package = "phase21" }
+phase2 = { git = "https://github.com/filecoin-project/phase2", branch = "blst", package = "phase21", default-features = false }
 simplelog = "0.8.0"
 rand_chacha = "0.2.1"
-dialoguer = "0.6.2"
-generic-array = "0.13.2"
+dialoguer = "0.7.1"
+generic-array = "0.14.4"
 structopt = "0.3.12"
 humansize = "1.1.0"
-indicatif = "0.14.0"
+indicatif = "0.15.0"
 groupy = "0.3.0"
 
 [dependencies.reqwest]
@@ -65,12 +64,14 @@ failure = "0.1.7"
 tempfile = "3"
 
 [features]
-default = ["gpu"]
+default = ["gpu", "pairing"]
 cpu-profile = ["gperftools"]
 heap-profile = ["gperftools/heap"]
 simd = ["storage-proofs/simd"]
 asm = ["storage-proofs/asm"]
 gpu = ["storage-proofs/gpu", "bellperson/gpu"]
+pairing = ["storage-proofs/pairing", "bellperson/pairing", "phase2/pairing"]
+blst = ["storage-proofs/blst", "bellperson/blst", "phase2/blst"]
 
 [[bench]]
 name = "preprocessing"

filecoin-proofs/src/api/mod.rs

@@ -587,8 +587,8 @@ where
 mod tests {
     use super::*;
 
+    use bellperson::bls::Fr;
     use ff::Field;
-    use paired::bls12_381::Fr;
     use rand::SeedableRng;
     use rand_xorshift::XorShiftRng;
     use storage_proofs::fr32::bytes_into_fr;

filecoin-proofs/src/api/post.rs

@@ -19,7 +19,6 @@ use storage_proofs::post::fallback;
 use storage_proofs::post::fallback::SectorProof;
 use storage_proofs::proof::ProofScheme;
 use storage_proofs::sector::*;
-use storage_proofs::settings;
 use storage_proofs::util::default_rows_to_discard;
 
 use crate::api::util::{as_safe_commitment, get_base_tree_leafs, get_base_tree_size};
@@ -624,22 +623,7 @@ pub fn verify_winning_post<Tree: 'static + MerkleTreeTrait>(
         k: None,
     };
 
-    let use_fil_blst = settings::SETTINGS.use_fil_blst;
-
-    let is_valid = if use_fil_blst {
-        info!("verify_winning_post: use_fil_blst=true");
-        let verifying_key_path = post_config.get_cache_verifying_key_path::<Tree>()?;
-        fallback::FallbackPoStCompound::verify_blst(
-            &pub_params,
-            &pub_inputs,
-            &proof,
-            proof.len() / 192,
-            &fallback::ChallengeRequirements {
-                minimum_challenge_count: post_config.challenge_count * post_config.sector_count,
-            },
-            &verifying_key_path,
-        )?
-    } else {
+    let is_valid = {
         let verifying_key = get_post_verifying_key::<Tree>(&post_config)?;
 
         let single_proof = MultiProof::new_from_reader(None, &proof[..], &verifying_key)?;
@@ -997,22 +981,7 @@ pub fn verify_window_post<Tree: 'static + MerkleTreeTrait>(
         k: None,
     };
 
-    let use_fil_blst = settings::SETTINGS.use_fil_blst;
-
-    let is_valid = if use_fil_blst {
-        info!("verify_window_post: use_fil_blst=true");
-        let verifying_key_path = post_config.get_cache_verifying_key_path::<Tree>()?;
-        fallback::FallbackPoStCompound::verify_blst(
-            &pub_params,
-            &pub_inputs,
-            &proof,
-            proof.len() / 192,
-            &fallback::ChallengeRequirements {
-                minimum_challenge_count: post_config.challenge_count * post_config.sector_count,
-            },
-            &verifying_key_path,
-        )?
-    } else {
+    let is_valid = {
         let verifying_key = get_post_verifying_key::<Tree>(&post_config)?;
         let multi_proof = MultiProof::new_from_reader(partitions, &proof[..], &verifying_key)?;
 

filecoin-proofs/src/api/seal.rs

@@ -3,11 +3,11 @@ use std::io::prelude::*;
 use std::path::{Path, PathBuf};
 
 use anyhow::{ensure, Context, Result};
+use bellperson::bls::Fr;
 use bincode::{deserialize, serialize};
 use log::{info, trace};
 use memmap::MmapOptions;
 use merkletree::store::{DiskStore, Store, StoreConfig};
-use paired::bls12_381::Fr;
 use storage_proofs::cache_key::CacheKey;
 use storage_proofs::compound_proof::{self, CompoundProof};
 use storage_proofs::drgraph::Graph;
@@ -21,7 +21,6 @@ use storage_proofs::porep::stacked::{
 };
 use storage_proofs::proof::ProofScheme;
 use storage_proofs::sector::SectorId;
-use storage_proofs::settings;
 use storage_proofs::util::default_rows_to_discard;
 
 use crate::api::util::{
@@ -505,7 +504,7 @@ pub fn seal_commit_phase2<Tree: 'static + MerkleTreeTrait>(
     )?;
     info!("snark_proof:finish");
 
-    let proof = MultiProof::new(groth_proofs, &groth_params.vk);
+    let proof = MultiProof::new(groth_proofs, &groth_params.pvk);
 
     let mut buf = Vec::with_capacity(
         SINGLE_PARTITION_PROOF_LEN * usize::from(PoRepProofPartitions::from(porep_config)),
@@ -609,27 +608,7 @@ pub fn verify_seal<Tree: 'static + MerkleTreeTrait>(
             k: None,
         };
 
-    let use_fil_blst = settings::SETTINGS.use_fil_blst;
-
-    let result = if use_fil_blst {
-        info!("verify_seal: use_fil_blst=true");
-        let verifying_key_path = porep_config.get_cache_verifying_key_path::<Tree>()?;
-
-        StackedCompound::verify_blst(
-            &compound_public_params,
-            &public_inputs,
-            &proof_vec,
-            proof_vec.len() / 192,
-            &ChallengeRequirements {
-                minimum_challenges: *POREP_MINIMUM_CHALLENGES
-                    .read()
-                    .expect("POREP_MINIMUM_CHALLENGES poisoned")
-                    .get(&u64::from(SectorSize::from(porep_config)))
-                    .expect("unknown sector size") as usize,
-            },
-            &verifying_key_path,
-        )
-    } else {
+    let result = {
         let sector_bytes = PaddedBytesAmount::from(porep_config);
         let verifying_key = get_stacked_verifying_key::<Tree>(porep_config)?;
 

filecoin-proofs/src/api/util.rs

@@ -1,6 +1,6 @@
 use anyhow::{Context, Result};
+use bellperson::bls::Fr;
 use merkletree::merkle::{get_merkle_tree_leafs, get_merkle_tree_len};
-use paired::bls12_381::Fr;
 use storage_proofs::fr32::{bytes_into_fr, fr_into_bytes};
 use storage_proofs::hasher::{Domain, Hasher};
 use storage_proofs::merkle::{get_base_tree_count, MerkleTreeTrait};

filecoin-proofs/src/bin/circuitinfo.rs

@@ -4,15 +4,14 @@ use log::{info, warn};
 use structopt::StructOpt;
 
 use bellperson::util_cs::bench_cs::BenchCS;
-use bellperson::Circuit;
+use bellperson::{bls::Bls12, Circuit};
 use filecoin_proofs::constants::*;
 use filecoin_proofs::parameters::{
     public_params, window_post_public_params, winning_post_public_params,
 };
 use filecoin_proofs::types::*;
 use filecoin_proofs::with_shape;
 use filecoin_proofs::PoStType;
-use paired::bls12_381::Bls12;
 use storage_proofs::compound_proof::CompoundProof;
 use storage_proofs::porep::stacked::{StackedCompound, StackedDrg};
 use storage_proofs::post::fallback::{FallbackPoSt, FallbackPoStCircuit, FallbackPoStCompound};

filecoin-proofs/src/bin/phase2.rs

@@ -9,6 +9,7 @@ use std::sync::mpsc::{channel, TryRecvError};
 use std::thread::{self, JoinHandle};
 use std::time::{Duration, Instant};
 
+use bellperson::bls::{Bls12, G1Affine, G1Uncompressed, G2Affine, G2Uncompressed};
 use bellperson::groth16;
 use byteorder::{BigEndian, ReadBytesExt};
 use clap::{App, AppSettings, Arg, ArgGroup, SubCommand};
@@ -22,7 +23,6 @@ use filecoin_proofs::types::{
 use filecoin_proofs::with_shape;
 use groupy::{CurveAffine, EncodedPoint};
 use log::{error, info, warn};
-use paired::bls12_381::{Bls12, G1Affine, G1Uncompressed, G2Affine, G2Uncompressed};
 use phase2::small::{read_small_params_from_large_file, MPCSmall, Streamer};
 use phase2::MPCParameters;
 use rand::rngs::OsRng;