feat: non-interactive PoRep (#1734)

Adding non-interactive PoRep. It's similar to the interactive PoRep, but
it performance 12.6x the amount of proofs. In order to not overwhelm the
system with too many parallel proofs, they are split into batches of 10,
which then leads to a similar resource consumption per batch as for a
single interactive PoRep.

fil-proofs-tooling/src/shared.rs

@@ -339,4 +339,5 @@ pub fn get_porep_config(
 ) -> PoRepConfig {
     let arbitrary_porep_id = [99; 32];
     PoRepConfig::new_groth16_with_features(sector_size, arbitrary_porep_id, api_version, features)
+        .expect("cannot set PoRep config")
 }

filecoin-proofs/src/api/seal.rs

@@ -33,7 +33,6 @@ use storage_proofs_porep::stacked::{
 use storage_proofs_update::vanilla::prepare_tree_r_data;
 use typenum::{Unsigned, U11, U2};
 
-use crate::POREP_MINIMUM_CHALLENGES;
 use crate::{
     api::util::{get_aggregate_target_len, pad_inputs_to_target, pad_proofs_to_target},
     api::{as_safe_commitment, commitment_from_fr, get_base_tree_leafs, get_base_tree_size, util},
@@ -568,9 +567,20 @@ pub fn seal_commit_phase2<Tree: 'static + MerkleTreeTrait>(
 
     proof.write(&mut buf)?;
 
+    // Non-interactive PoRep is an aggregated proof, hence we use that as the returned buffer.
+    if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) {
+        buf = aggregate_seal_commit_proofs::<Tree>(
+            porep_config,
+            &[comm_r],
+            &[seed],
+            &[SealCommitOutput { proof: buf }],
+            groth16::aggregate::AggregateVersion::V2,
+        )?;
+    }
+
     // Verification is cheap when parameters are cached,
     // and it is never correct to return a proof which does not verify.
-    verify_seal::<Tree>(
+    let is_valid = verify_seal::<Tree>(
         porep_config,
         comm_r,
         comm_d,
@@ -581,11 +591,10 @@ pub fn seal_commit_phase2<Tree: 'static + MerkleTreeTrait>(
         &buf,
     )
     .context("post-seal verification sanity check failed")?;
-
-    let out = SealCommitOutput { proof: buf };
+    ensure!(is_valid, "post seal aggregation verifies");
 
     info!("seal_commit_phase2:finish: {:?}", sector_id);
-    Ok(out)
+    Ok(SealCommitOutput { proof: buf })
 }
 
 /// Given the specified arguments, this method returns the inputs that were used to
@@ -892,6 +901,28 @@ pub fn verify_seal<Tree: 'static + MerkleTreeTrait>(
 ) -> Result<bool> {
     info!("verify_seal:start: {:?}", sector_id);
 
+    // Non-interactive PoReps are aggregated, but it should be possible to use the usual PoRep
+    // APIs, hence branch out here and not one layer higher.
+    if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) {
+        let inputs = get_seal_inputs::<Tree>(
+            porep_config,
+            comm_r_in,
+            comm_d_in,
+            prover_id,
+            sector_id,
+            ticket,
+            seed,
+        )?;
+        return verify_aggregate_seal_commit_proofs::<Tree>(
+            porep_config,
+            proof_vec.to_vec(),
+            &[comm_r_in],
+            &[seed],
+            inputs,
+            groth16::aggregate::AggregateVersion::V2,
+        );
+    }
+
     ensure!(comm_d_in != [0; 32], "Invalid all zero commitment (comm_d)");
     ensure!(comm_r_in != [0; 32], "Invalid all zero commitment (comm_r)");
     ensure!(!proof_vec.is_empty(), "Invalid proof bytes (empty vector)");
@@ -946,8 +977,7 @@ pub fn verify_seal<Tree: 'static + MerkleTreeTrait>(
             &public_inputs,
             &proof,
             &ChallengeRequirements {
-                minimum_challenges: POREP_MINIMUM_CHALLENGES
-                    .from_sector_size(u64::from(porep_config.sector_size)),
+                minimum_challenges: porep_config.minimum_challenges(),
             },
         )
     };
@@ -1061,8 +1091,7 @@ pub fn verify_batch_seal<Tree: 'static + MerkleTreeTrait>(
         &public_inputs,
         &proofs,
         &ChallengeRequirements {
-            minimum_challenges: POREP_MINIMUM_CHALLENGES
-                .from_sector_size(u64::from(porep_config.sector_size)),
+            minimum_challenges: porep_config.minimum_challenges(),
         },
     )
     .map_err(Into::into);

filecoin-proofs/src/constants.rs

@@ -1,5 +1,5 @@
+use std::collections::HashMap;
 use std::sync::RwLock;
-use std::{collections::HashMap, sync::RwLockWriteGuard};
 
 pub use storage_proofs_core::drgraph::BASE_DEGREE as DRG_DEGREE;
 pub use storage_proofs_porep::stacked::EXP_DEGREE;
@@ -47,47 +47,7 @@ pub const PUBLISHED_SECTOR_SIZES: [u64; 10] = [
     SECTOR_SIZE_64_GIB,
 ];
 
-pub struct PorepMinimumChallenges(RwLock<HashMap<u64, usize>>);
-impl PorepMinimumChallenges {
-    fn new() -> Self {
-        Self(RwLock::new(
-            [
-                (SECTOR_SIZE_2_KIB, 2),
-                (SECTOR_SIZE_4_KIB, 2),
-                (SECTOR_SIZE_16_KIB, 2),
-                (SECTOR_SIZE_32_KIB, 2),
-                (SECTOR_SIZE_8_MIB, 2),
-                (SECTOR_SIZE_16_MIB, 2),
-                (SECTOR_SIZE_512_MIB, 2),
-                (SECTOR_SIZE_1_GIB, 2),
-                (SECTOR_SIZE_32_GIB, 176),
-                (SECTOR_SIZE_64_GIB, 176),
-            ]
-            .iter()
-            .copied()
-            .collect(),
-        ))
-    }
-
-    pub fn get_mut(&self) -> RwLockWriteGuard<'_, HashMap<u64, usize>> {
-        self.0.write().expect("POREP_MINIMUM_CHALLENGES poisoned")
-    }
-
-    pub fn from_sector_size(&self, sector_size: u64) -> usize {
-        match self
-            .0
-            .read()
-            .expect("POREP_MINIMUM_CHALLENGES poisoned")
-            .get(&sector_size)
-        {
-            Some(c) => *c,
-            None => panic!("invalid sector size"),
-        }
-    }
-}
-
 lazy_static! {
-    pub static ref POREP_MINIMUM_CHALLENGES: PorepMinimumChallenges = PorepMinimumChallenges::new();
     pub static ref POREP_PARTITIONS: RwLock<HashMap<u64, u8>> = RwLock::new(
         [
             (SECTOR_SIZE_2_KIB, 1),
@@ -144,6 +104,43 @@ lazy_static! {
     );
 }
 
+/// Returns the minimum number of challenges used for the (synth and non-synth) interactive PoRep
+/// for a certain sector size.
+pub(crate) const fn get_porep_interactive_minimum_challenges(sector_size: u64) -> usize {
+    match sector_size {
+        SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB
+        | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 2,
+        SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 176,
+        _ => panic!("invalid sector size"),
+    }
+}
+
+/// Returns the minimum number of challenges used for the non-interactive PoRep fo a certain sector
+/// size.
+pub(crate) const fn get_porep_non_interactive_minimum_challenges(sector_size: u64) -> usize {
+    match sector_size {
+        SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB
+        | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 4,
+        SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 2253,
+        _ => panic!("invalid sector size"),
+    }
+}
+
+/// Returns the number of partitions for non-interactive PoRep for a certain sector size.
+pub const fn get_porep_non_interactive_partitions(sector_size: u64) -> u8 {
+    match sector_size {
+        // The filename of the parameter files and verifying keys depend on the number of
+        // challenges per partition. In order to be able to re-use the files that were generated
+        // for the interactive PoRep, we need to use certain numbers, also for the test sector
+        // sizes. The number of challenges per partition for test sizes is 2, for production
+        // parameters it's 18.
+        SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB
+        | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 2,
+        SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 126,
+        _ => panic!("invalid sector size"),
+    }
+}
+
 /// The size of a single snark proof.
 pub const SINGLE_PARTITION_PROOF_LEN: usize = 192;
 
@@ -156,6 +153,9 @@ pub const MINIMUM_RESERVED_BYTES_FOR_PIECE_IN_FULLY_ALIGNED_SECTOR: u64 =
 /// The minimum size a single piece must have before padding.
 pub const MIN_PIECE_SIZE: UnpaddedBytesAmount = UnpaddedBytesAmount(127);
 
+/// The maximum number of challenges per partition the circuits can work with.
+pub(crate) const MAX_CHALLENGES_PER_PARTITION: u8 = 18;
+
 /// The hasher used for creating comm_d.
 pub type DefaultPieceHasher = Sha256Hasher;
 pub type DefaultPieceDomain = <DefaultPieceHasher as Hasher>::Domain;

filecoin-proofs/src/parameters.rs

@@ -4,9 +4,8 @@ use storage_proofs_porep::stacked::{self, Challenges, StackedDrg};
 use storage_proofs_post::fallback::{self, FallbackPoSt};
 
 use crate::{
-    constants::{DefaultPieceHasher, DRG_DEGREE, EXP_DEGREE, LAYERS},
+    constants::{DefaultPieceHasher, DRG_DEGREE, EXP_DEGREE, LAYERS, MAX_CHALLENGES_PER_PARTITION},
     types::{MerkleTreeTrait, PoRepConfig, PoStConfig},
-    POREP_MINIMUM_CHALLENGES,
 };
 
 type WinningPostSetupParams = fallback::SetupParams;
@@ -68,12 +67,11 @@ pub fn window_post_setup_params(post_config: &PoStConfig) -> WindowPostSetupPara
 }
 
 pub fn setup_params(porep_config: &PoRepConfig) -> Result<stacked::SetupParams> {
-    let use_synthetic = porep_config.feature_enabled(ApiFeature::SyntheticPoRep);
     let sector_bytes = porep_config.padded_bytes_amount();
     let challenges = select_challenges(
         usize::from(porep_config.partitions),
-        POREP_MINIMUM_CHALLENGES.from_sector_size(u64::from(sector_bytes)),
-        use_synthetic,
+        porep_config.minimum_challenges(),
+        &porep_config.api_features,
     );
     let num_layers = *LAYERS
         .read()
@@ -113,11 +111,15 @@ const fn div_ceil(x: usize, y: usize) -> usize {
 fn select_challenges(
     partitions: usize,
     minimum_total_challenges: usize,
-    use_synthetic: bool,
+    features: &[ApiFeature],
 ) -> Challenges {
     let challenges = div_ceil(minimum_total_challenges, partitions);
-    if use_synthetic {
+    assert!(challenges <= usize::from(MAX_CHALLENGES_PER_PARTITION));
+
+    if features.contains(&ApiFeature::SyntheticPoRep) {
         Challenges::new_synthetic(challenges)
+    } else if features.contains(&ApiFeature::NonInteractivePoRep) {
+        Challenges::new_non_interactive(challenges)
     } else {
         Challenges::new_interactive(challenges)
     }
@@ -131,8 +133,7 @@ mod tests {
 
     #[test]
     fn partition_layer_challenges_test() {
-        let f =
-            |partitions| select_challenges(partitions, 12, false).num_challenges_per_partition();
+        let f = |partitions| select_challenges(partitions, 12, &[]).num_challenges_per_partition();
         // Update to ensure all supported PoRepProofPartitions options are represented here.
         assert_eq!(6, f(usize::from(PoRepProofPartitions(2))));
 

filecoin-proofs/src/types/porep_config.rs

@@ -1,6 +1,6 @@
 use std::path::PathBuf;
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use storage_proofs_core::{
     api_version::{ApiFeature, ApiVersion},
     merkle::MerkleTreeTrait,
@@ -12,7 +12,7 @@ use storage_proofs_core::{
 use storage_proofs_porep::stacked::{StackedCircuit, StackedCompound};
 
 use crate::{
-    constants::DefaultPieceHasher,
+    constants::{self, DefaultPieceHasher},
     parameters::public_params,
     types::{PaddedBytesAmount, PoRepProofPartitions, SectorSize, UnpaddedBytesAmount},
     POREP_PARTITIONS,
@@ -78,8 +78,8 @@ impl PoRepConfig {
         porep_id: [u8; 32],
         api_version: ApiVersion,
         api_features: Vec<ApiFeature>,
-    ) -> Self {
-        Self {
+    ) -> Result<Self> {
+        let mut config = Self {
             sector_size: SectorSize(sector_size),
             partitions: PoRepProofPartitions(
                 *POREP_PARTITIONS
@@ -90,15 +90,46 @@ impl PoRepConfig {
             ),
             porep_id,
             api_version,
-            api_features,
+            api_features: vec![],
+        };
+        for feature in api_features {
+            config.enable_feature(feature)?;
         }
+        Ok(config)
     }
 
     #[inline]
-    pub fn enable_feature(&mut self, feat: ApiFeature) {
+    pub fn enable_feature(&mut self, feat: ApiFeature) -> Result<()> {
+        for conflict in feat.conflicting_features() {
+            if self.feature_enabled(*conflict) {
+                return Err(anyhow!(
+                    "Cannot enable feature `{feat}` when `{conflict}` is already enabled"
+                ));
+            }
+        }
+
+        match feat {
+            ApiFeature::SyntheticPoRep => {
+                self.partitions = PoRepProofPartitions(
+                    *POREP_PARTITIONS
+                        .read()
+                        .expect("POREP_PARTITIONS poisoned")
+                        .get(&self.sector_size.into())
+                        .expect("unknown sector size"),
+                );
+            }
+            ApiFeature::NonInteractivePoRep => {
+                self.partitions = PoRepProofPartitions(
+                    constants::get_porep_non_interactive_partitions(self.sector_size.into()),
+                );
+            }
+        }
+
         if !self.feature_enabled(feat) {
             self.api_features.push(feat);
         }
+
+        Ok(())
     }
 
     #[inline]
@@ -116,6 +147,14 @@ impl PoRepConfig {
         self.padded_bytes_amount().into()
     }
 
+    pub fn minimum_challenges(&self) -> usize {
+        if self.feature_enabled(ApiFeature::NonInteractivePoRep) {
+            constants::get_porep_non_interactive_minimum_challenges(u64::from(self.sector_size))
+        } else {
+            constants::get_porep_interactive_minimum_challenges(u64::from(self.sector_size))
+        }
+    }
+
     /// Returns the cache identifier as used by `storage-proofs::parameter_cache`.
     pub fn get_cache_identifier<Tree: 'static + MerkleTreeTrait>(&self) -> Result<String> {
         let params = public_params::<Tree>(self)?;