diff --git a/fil-proofs-tooling/src/shared.rs b/fil-proofs-tooling/src/shared.rs index 09b4c508e..2124f0940 100644 --- a/fil-proofs-tooling/src/shared.rs +++ b/fil-proofs-tooling/src/shared.rs @@ -339,4 +339,5 @@ pub fn get_porep_config( ) -> PoRepConfig { let arbitrary_porep_id = [99; 32]; PoRepConfig::new_groth16_with_features(sector_size, arbitrary_porep_id, api_version, features) + .expect("cannot set PoRep config") } diff --git a/filecoin-proofs/src/api/seal.rs b/filecoin-proofs/src/api/seal.rs index 246bb85bc..d77204680 100644 --- a/filecoin-proofs/src/api/seal.rs +++ b/filecoin-proofs/src/api/seal.rs @@ -33,7 +33,6 @@ use storage_proofs_porep::stacked::{ use storage_proofs_update::vanilla::prepare_tree_r_data; use typenum::{Unsigned, U11, U2}; -use crate::POREP_MINIMUM_CHALLENGES; use crate::{ api::util::{get_aggregate_target_len, pad_inputs_to_target, pad_proofs_to_target}, api::{as_safe_commitment, commitment_from_fr, get_base_tree_leafs, get_base_tree_size, util}, @@ -568,9 +567,20 @@ pub fn seal_commit_phase2( proof.write(&mut buf)?; + // Non-interactive PoRep is an aggregated proof, hence we use that as the returned buffer. + if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) { + buf = aggregate_seal_commit_proofs::( + porep_config, + &[comm_r], + &[seed], + &[SealCommitOutput { proof: buf }], + groth16::aggregate::AggregateVersion::V2, + )?; + } + // Verification is cheap when parameters are cached, // and it is never correct to return a proof which does not verify. - verify_seal::( + let is_valid = verify_seal::( porep_config, comm_r, comm_d, @@ -581,11 +591,10 @@ pub fn seal_commit_phase2( &buf, ) .context("post-seal verification sanity check failed")?; - - let out = SealCommitOutput { proof: buf }; + ensure!(is_valid, "post seal aggregation verifies"); info!("seal_commit_phase2:finish: {:?}", sector_id); - Ok(out) + Ok(SealCommitOutput { proof: buf }) } /// Given the specified arguments, this method returns the inputs that were used to @@ -892,6 +901,28 @@ pub fn verify_seal( ) -> Result { info!("verify_seal:start: {:?}", sector_id); + // Non-interactive PoReps are aggregated, but it should be possible to use the usual PoRep + // APIs, hence branch out here and not one layer higher. + if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) { + let inputs = get_seal_inputs::( + porep_config, + comm_r_in, + comm_d_in, + prover_id, + sector_id, + ticket, + seed, + )?; + return verify_aggregate_seal_commit_proofs::( + porep_config, + proof_vec.to_vec(), + &[comm_r_in], + &[seed], + inputs, + groth16::aggregate::AggregateVersion::V2, + ); + } + ensure!(comm_d_in != [0; 32], "Invalid all zero commitment (comm_d)"); ensure!(comm_r_in != [0; 32], "Invalid all zero commitment (comm_r)"); ensure!(!proof_vec.is_empty(), "Invalid proof bytes (empty vector)"); @@ -946,8 +977,7 @@ pub fn verify_seal( &public_inputs, &proof, &ChallengeRequirements { - minimum_challenges: POREP_MINIMUM_CHALLENGES - .from_sector_size(u64::from(porep_config.sector_size)), + minimum_challenges: porep_config.minimum_challenges(), }, ) }; @@ -1061,8 +1091,7 @@ pub fn verify_batch_seal( &public_inputs, &proofs, &ChallengeRequirements { - minimum_challenges: POREP_MINIMUM_CHALLENGES - .from_sector_size(u64::from(porep_config.sector_size)), + minimum_challenges: porep_config.minimum_challenges(), }, ) .map_err(Into::into); diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs index 31e5b64ab..a96986089 100644 --- a/filecoin-proofs/src/constants.rs +++ b/filecoin-proofs/src/constants.rs @@ -1,5 +1,5 @@ +use std::collections::HashMap; use std::sync::RwLock; -use std::{collections::HashMap, sync::RwLockWriteGuard}; pub use storage_proofs_core::drgraph::BASE_DEGREE as DRG_DEGREE; pub use storage_proofs_porep::stacked::EXP_DEGREE; @@ -47,47 +47,7 @@ pub const PUBLISHED_SECTOR_SIZES: [u64; 10] = [ SECTOR_SIZE_64_GIB, ]; -pub struct PorepMinimumChallenges(RwLock>); -impl PorepMinimumChallenges { - fn new() -> Self { - Self(RwLock::new( - [ - (SECTOR_SIZE_2_KIB, 2), - (SECTOR_SIZE_4_KIB, 2), - (SECTOR_SIZE_16_KIB, 2), - (SECTOR_SIZE_32_KIB, 2), - (SECTOR_SIZE_8_MIB, 2), - (SECTOR_SIZE_16_MIB, 2), - (SECTOR_SIZE_512_MIB, 2), - (SECTOR_SIZE_1_GIB, 2), - (SECTOR_SIZE_32_GIB, 176), - (SECTOR_SIZE_64_GIB, 176), - ] - .iter() - .copied() - .collect(), - )) - } - - pub fn get_mut(&self) -> RwLockWriteGuard<'_, HashMap> { - self.0.write().expect("POREP_MINIMUM_CHALLENGES poisoned") - } - - pub fn from_sector_size(&self, sector_size: u64) -> usize { - match self - .0 - .read() - .expect("POREP_MINIMUM_CHALLENGES poisoned") - .get(§or_size) - { - Some(c) => *c, - None => panic!("invalid sector size"), - } - } -} - lazy_static! { - pub static ref POREP_MINIMUM_CHALLENGES: PorepMinimumChallenges = PorepMinimumChallenges::new(); pub static ref POREP_PARTITIONS: RwLock> = RwLock::new( [ (SECTOR_SIZE_2_KIB, 1), @@ -144,6 +104,43 @@ lazy_static! { ); } +/// Returns the minimum number of challenges used for the (synth and non-synth) interactive PoRep +/// for a certain sector size. +pub(crate) const fn get_porep_interactive_minimum_challenges(sector_size: u64) -> usize { + match sector_size { + SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB + | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 2, + SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 176, + _ => panic!("invalid sector size"), + } +} + +/// Returns the minimum number of challenges used for the non-interactive PoRep fo a certain sector +/// size. +pub(crate) const fn get_porep_non_interactive_minimum_challenges(sector_size: u64) -> usize { + match sector_size { + SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB + | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 4, + SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 2253, + _ => panic!("invalid sector size"), + } +} + +/// Returns the number of partitions for non-interactive PoRep for a certain sector size. +pub const fn get_porep_non_interactive_partitions(sector_size: u64) -> u8 { + match sector_size { + // The filename of the parameter files and verifying keys depend on the number of + // challenges per partition. In order to be able to re-use the files that were generated + // for the interactive PoRep, we need to use certain numbers, also for the test sector + // sizes. The number of challenges per partition for test sizes is 2, for production + // parameters it's 18. + SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB + | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 2, + SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 126, + _ => panic!("invalid sector size"), + } +} + /// The size of a single snark proof. pub const SINGLE_PARTITION_PROOF_LEN: usize = 192; @@ -156,6 +153,9 @@ pub const MINIMUM_RESERVED_BYTES_FOR_PIECE_IN_FULLY_ALIGNED_SECTOR: u64 = /// The minimum size a single piece must have before padding. pub const MIN_PIECE_SIZE: UnpaddedBytesAmount = UnpaddedBytesAmount(127); +/// The maximum number of challenges per partition the circuits can work with. +pub(crate) const MAX_CHALLENGES_PER_PARTITION: u8 = 18; + /// The hasher used for creating comm_d. pub type DefaultPieceHasher = Sha256Hasher; pub type DefaultPieceDomain = ::Domain; diff --git a/filecoin-proofs/src/parameters.rs b/filecoin-proofs/src/parameters.rs index 4d97e98ad..01a2962e7 100644 --- a/filecoin-proofs/src/parameters.rs +++ b/filecoin-proofs/src/parameters.rs @@ -4,9 +4,8 @@ use storage_proofs_porep::stacked::{self, Challenges, StackedDrg}; use storage_proofs_post::fallback::{self, FallbackPoSt}; use crate::{ - constants::{DefaultPieceHasher, DRG_DEGREE, EXP_DEGREE, LAYERS}, + constants::{DefaultPieceHasher, DRG_DEGREE, EXP_DEGREE, LAYERS, MAX_CHALLENGES_PER_PARTITION}, types::{MerkleTreeTrait, PoRepConfig, PoStConfig}, - POREP_MINIMUM_CHALLENGES, }; type WinningPostSetupParams = fallback::SetupParams; @@ -68,12 +67,11 @@ pub fn window_post_setup_params(post_config: &PoStConfig) -> WindowPostSetupPara } pub fn setup_params(porep_config: &PoRepConfig) -> Result { - let use_synthetic = porep_config.feature_enabled(ApiFeature::SyntheticPoRep); let sector_bytes = porep_config.padded_bytes_amount(); let challenges = select_challenges( usize::from(porep_config.partitions), - POREP_MINIMUM_CHALLENGES.from_sector_size(u64::from(sector_bytes)), - use_synthetic, + porep_config.minimum_challenges(), + &porep_config.api_features, ); let num_layers = *LAYERS .read() @@ -113,11 +111,15 @@ const fn div_ceil(x: usize, y: usize) -> usize { fn select_challenges( partitions: usize, minimum_total_challenges: usize, - use_synthetic: bool, + features: &[ApiFeature], ) -> Challenges { let challenges = div_ceil(minimum_total_challenges, partitions); - if use_synthetic { + assert!(challenges <= usize::from(MAX_CHALLENGES_PER_PARTITION)); + + if features.contains(&ApiFeature::SyntheticPoRep) { Challenges::new_synthetic(challenges) + } else if features.contains(&ApiFeature::NonInteractivePoRep) { + Challenges::new_non_interactive(challenges) } else { Challenges::new_interactive(challenges) } @@ -131,8 +133,7 @@ mod tests { #[test] fn partition_layer_challenges_test() { - let f = - |partitions| select_challenges(partitions, 12, false).num_challenges_per_partition(); + let f = |partitions| select_challenges(partitions, 12, &[]).num_challenges_per_partition(); // Update to ensure all supported PoRepProofPartitions options are represented here. assert_eq!(6, f(usize::from(PoRepProofPartitions(2)))); diff --git a/filecoin-proofs/src/types/porep_config.rs b/filecoin-proofs/src/types/porep_config.rs index a923d95ed..2eff1b324 100644 --- a/filecoin-proofs/src/types/porep_config.rs +++ b/filecoin-proofs/src/types/porep_config.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use anyhow::Result; +use anyhow::{anyhow, Result}; use storage_proofs_core::{ api_version::{ApiFeature, ApiVersion}, merkle::MerkleTreeTrait, @@ -12,7 +12,7 @@ use storage_proofs_core::{ use storage_proofs_porep::stacked::{StackedCircuit, StackedCompound}; use crate::{ - constants::DefaultPieceHasher, + constants::{self, DefaultPieceHasher}, parameters::public_params, types::{PaddedBytesAmount, PoRepProofPartitions, SectorSize, UnpaddedBytesAmount}, POREP_PARTITIONS, @@ -78,8 +78,8 @@ impl PoRepConfig { porep_id: [u8; 32], api_version: ApiVersion, api_features: Vec, - ) -> Self { - Self { + ) -> Result { + let mut config = Self { sector_size: SectorSize(sector_size), partitions: PoRepProofPartitions( *POREP_PARTITIONS @@ -90,15 +90,46 @@ impl PoRepConfig { ), porep_id, api_version, - api_features, + api_features: vec![], + }; + for feature in api_features { + config.enable_feature(feature)?; } + Ok(config) } #[inline] - pub fn enable_feature(&mut self, feat: ApiFeature) { + pub fn enable_feature(&mut self, feat: ApiFeature) -> Result<()> { + for conflict in feat.conflicting_features() { + if self.feature_enabled(*conflict) { + return Err(anyhow!( + "Cannot enable feature `{feat}` when `{conflict}` is already enabled" + )); + } + } + + match feat { + ApiFeature::SyntheticPoRep => { + self.partitions = PoRepProofPartitions( + *POREP_PARTITIONS + .read() + .expect("POREP_PARTITIONS poisoned") + .get(&self.sector_size.into()) + .expect("unknown sector size"), + ); + } + ApiFeature::NonInteractivePoRep => { + self.partitions = PoRepProofPartitions( + constants::get_porep_non_interactive_partitions(self.sector_size.into()), + ); + } + } + if !self.feature_enabled(feat) { self.api_features.push(feat); } + + Ok(()) } #[inline] @@ -116,6 +147,14 @@ impl PoRepConfig { self.padded_bytes_amount().into() } + pub fn minimum_challenges(&self) -> usize { + if self.feature_enabled(ApiFeature::NonInteractivePoRep) { + constants::get_porep_non_interactive_minimum_challenges(u64::from(self.sector_size)) + } else { + constants::get_porep_interactive_minimum_challenges(u64::from(self.sector_size)) + } + } + /// Returns the cache identifier as used by `storage-proofs::parameter_cache`. pub fn get_cache_identifier(&self) -> Result { let params = public_params::(self)?; diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index b8e624fb3..cc024fc4b 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -11,8 +11,8 @@ use ff::Field; use filecoin_hashers::Hasher; use filecoin_proofs::{ add_piece, aggregate_empty_sector_update_proofs, aggregate_seal_commit_proofs, clear_cache, - clear_layer_data, clear_synthetic_proofs, compute_comm_d, decode_from, decode_from_range, - encode_into, fauxrep_aux, generate_empty_sector_update_proof, + clear_synthetic_proofs, compute_comm_d, decode_from, decode_from_range, encode_into, + fauxrep_aux, generate_empty_sector_update_proof, generate_empty_sector_update_proof_with_vanilla, generate_fallback_sector_challenges, generate_partition_proofs, generate_piece_commitment, generate_single_partition_proof, generate_single_vanilla_proof, generate_single_window_post_with_vanilla, generate_synth_proofs, @@ -174,7 +174,7 @@ fn test_seal_lifecycle_2kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -204,6 +204,11 @@ fn test_seal_lifecycle_upgrade_2kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + MAX_LEGACY_REGISTERED_SEAL_PROOF_ID + 1, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id_num, api_version, features) in test_inputs { @@ -213,7 +218,7 @@ fn test_seal_lifecycle_upgrade_2kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle_upgrade::(&porep_config)?; } @@ -233,6 +238,11 @@ fn test_seal_lifecycle_4kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -241,7 +251,7 @@ fn test_seal_lifecycle_4kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -261,6 +271,11 @@ fn test_seal_lifecycle_upgrade_4kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -269,7 +284,7 @@ fn test_seal_lifecycle_upgrade_4kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle_upgrade::(&porep_config)?; } @@ -289,6 +304,11 @@ fn test_seal_lifecycle_16kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -297,7 +317,7 @@ fn test_seal_lifecycle_16kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -317,6 +337,11 @@ fn test_seal_lifecycle_upgrade_16kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -325,7 +350,7 @@ fn test_seal_lifecycle_upgrade_16kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle_upgrade::(&porep_config)?; } @@ -345,6 +370,11 @@ fn test_seal_lifecycle_32kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -353,7 +383,7 @@ fn test_seal_lifecycle_32kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -436,9 +466,30 @@ fn test_seal_lifecycle_32gib_porep_id_v1_2_top_8_8_0_api_v1_2() -> Result<()> { let porep_id = to_porep_id_verified(porep_id_v1_2, ApiVersion::V1_2_0); assert!(!is_legacy_porep_id(porep_id)); - let mut porep_config = - PoRepConfig::new_groth16(SECTOR_SIZE_32_GIB, porep_id, ApiVersion::V1_2_0); - porep_config.enable_feature(ApiFeature::SyntheticPoRep); + let porep_config = PoRepConfig::new_groth16_with_features( + SECTOR_SIZE_32_GIB, + porep_id, + ApiVersion::V1_2_0, + vec![ApiFeature::SyntheticPoRep], + )?; + + seal_lifecycle::(&porep_config) +} + +#[cfg(feature = "big-tests")] +#[test] +fn test_seal_lifecycle_32gib_porep_id_v1_2_ni_top_8_8_0_api_v1_2() -> Result<()> { + let porep_id_v1_2: u64 = 8; // This is a RegisteredSealProof value + + let porep_id = to_porep_id_verified(porep_id_v1_2, ApiVersion::V1_2_0); + assert!(!is_legacy_porep_id(porep_id)); + + let porep_config = PoRepConfig::new_groth16_with_features( + SECTOR_SIZE_32_GIB, + porep_id, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + )?; seal_lifecycle::(&porep_config) } @@ -493,7 +544,25 @@ fn test_seal_lifecycle_64gib_porep_id_v1_2_top_8_8_2_api_v1_2() -> Result<()> { porep_id, ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], - ); + )?; + + seal_lifecycle::(&porep_config) +} + +#[cfg(feature = "big-tests")] +#[test] +fn test_seal_lifecycle_64gib_porep_id_v1_2_ni_top_8_8_2_api_v1_2() -> Result<()> { + let porep_id_v1_2: u64 = 9; // This is a RegisteredSealProof value + + let porep_id = to_porep_id_verified(porep_id_v1_2, ApiVersion::V1_2_0); + assert!(!is_legacy_porep_id(porep_id)); + + let porep_config = PoRepConfig::new_groth16_with_features( + SECTOR_SIZE_64_GIB, + porep_id, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + )?; seal_lifecycle::(&porep_config) } @@ -2570,7 +2639,7 @@ fn create_seal_for_upgrade_aggregation< pre_commit_output, &piece_infos, )?; - clear_layer_data::(cache_dir.path())?; + clear_cache::(cache_dir.path())?; } else { info!("SyntheticPoRep is NOT enabled"); validate_cache_for_commit::<_, _, Tree>(cache_dir.path(), sealed_sector_file.path())?; diff --git a/storage-proofs-core/src/api_version.rs b/storage-proofs-core/src/api_version.rs index 5576b3d67..a0966b54f 100644 --- a/storage-proofs-core/src/api_version.rs +++ b/storage-proofs-core/src/api_version.rs @@ -95,22 +95,41 @@ impl FromStr for ApiVersion { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ApiFeature { SyntheticPoRep, + NonInteractivePoRep, } impl ApiFeature { #[inline] pub fn first_supported_version(&self) -> ApiVersion { match self { - ApiFeature::SyntheticPoRep => ApiVersion::V1_2_0, + ApiFeature::SyntheticPoRep | ApiFeature::NonInteractivePoRep => ApiVersion::V1_2_0, } } #[inline] pub fn last_supported_version(&self) -> Option { match self { - ApiFeature::SyntheticPoRep => None, + ApiFeature::SyntheticPoRep | ApiFeature::NonInteractivePoRep => None, } } + + /// Return the features that are in conflict with the current one. + pub fn conflicting_features(&self) -> &[ApiFeature] { + match self { + ApiFeature::SyntheticPoRep => &[ApiFeature::NonInteractivePoRep], + ApiFeature::NonInteractivePoRep => &[ApiFeature::SyntheticPoRep], + } + } +} + +impl Display for ApiFeature { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let api_feature_str = match self { + Self::SyntheticPoRep => "synthetic-porep", + Self::NonInteractivePoRep => "non-interactive-porep", + }; + write!(f, "{}", api_feature_str) + } } #[test] @@ -145,3 +164,10 @@ fn test_api_feature_synthetic_porep() { assert!(feature.first_supported_version() == ApiVersion::V1_2_0); assert!(feature.last_supported_version().is_none()); } + +#[test] +fn test_api_feature_non_interactive_porep() { + let feature = ApiFeature::NonInteractivePoRep; + assert!(feature.first_supported_version() == ApiVersion::V1_2_0); + assert!(feature.last_supported_version().is_none()); +} diff --git a/storage-proofs-core/src/compound_proof.rs b/storage-proofs-core/src/compound_proof.rs index ccc18e455..7ba81fce4 100644 --- a/storage-proofs-core/src/compound_proof.rs +++ b/storage-proofs-core/src/compound_proof.rs @@ -1,3 +1,5 @@ +use std::cmp; + use anyhow::{ensure, Context}; use bellperson::{ groth16::{ @@ -25,6 +27,12 @@ use crate::{ proof::ProofScheme, }; +/// The maximum number of Groth16 proofs that will be processed in parallel. This limit is set as +/// synthesis takes a lot of memory. The current value is based on the number of proofs that are +/// run in parallel in the interactive PoRep (the number of partitions). This way there's just a +/// single batch for the interactive PoRep, but the non-interactive PoRep is split into batches. +const MAX_GROTH16_BATCH_SIZE: usize = 10; + #[derive(Clone)] pub struct SetupParams<'a, S: ProofScheme<'a>> { pub vanilla_params: >::SetupParams, @@ -242,7 +250,7 @@ where "cannot create a circuit proof over missing vanilla proofs" ); - let circuits = vanilla_proofs + let mut circuits = vanilla_proofs .into_par_iter() .enumerate() .map(|(k, vanilla_proof)| { @@ -256,14 +264,26 @@ where }) .collect::>>()?; - let groth_proofs = if priority { - create_random_proof_batch_in_priority(circuits, groth_params, &mut rng)? + // The same function is used within the while loop below, decide on the function once and + // not on every iteration. + let create_random_proof_batch_fun = if priority { + create_random_proof_batch_in_priority } else { - create_random_proof_batch(circuits, groth_params, &mut rng)? + create_random_proof_batch }; + let mut groth_proofs = Vec::with_capacity(circuits.len()); + // Bellperson expects a vector of proofs, hence drain it from the list of proofs, so that + // we don't need to keep an extra copy around. + while !circuits.is_empty() { + let size = cmp::min(MAX_GROTH16_BATCH_SIZE, circuits.len()); + let batch = circuits.drain(0..size).collect(); + let proofs = create_random_proof_batch_fun(batch, groth_params, &mut rng)?; + groth_proofs.extend_from_slice(&proofs); + } + groth_proofs - .into_iter() + .iter() .map(|groth_proof| { let mut proof_vec = Vec::new(); groth_proof.write(&mut proof_vec)?; diff --git a/storage-proofs-porep/src/stacked/vanilla/challenges.rs b/storage-proofs-porep/src/stacked/vanilla/challenges.rs index e520af508..9214a79c5 100644 --- a/storage-proofs-porep/src/stacked/vanilla/challenges.rs +++ b/storage-proofs-porep/src/stacked/vanilla/challenges.rs @@ -119,10 +119,48 @@ impl SynthChallenges { } } +#[derive(Clone, Debug)] +pub struct NiChallenges { + challenges_per_partition: usize, +} + +impl NiChallenges { + pub const fn new(challenges_per_partition: usize) -> Self { + Self { + challenges_per_partition, + } + } + + pub fn derive( + &self, + sector_nodes: usize, + replica_id: &D, + comm_r: &D, + k: u8, + ) -> Vec { + const TAG: &[u8] = b"filecoin.io|PoRep|1|NonInteractive|1"; + let hash_init = Sha256::new() + .chain_update(TAG) + .chain_update(replica_id.into_bytes()) + .chain_update(comm_r); + (0..self.challenges_per_partition) + .map(|i| { + let j: u32 = ((self.challenges_per_partition * k as usize) + i) as u32; + + let hash = hash_init.clone().chain_update(j.to_le_bytes()).finalize(); + + let bigint = BigUint::from_bytes_le(hash.as_ref()); + bigint_to_challenge(bigint, sector_nodes) + }) + .collect() + } +} + #[derive(Clone, Debug)] pub enum Challenges { Interactive(InteractiveChallenges), Synth(SynthChallenges), + Ni(NiChallenges), } impl Challenges { @@ -134,6 +172,10 @@ impl Challenges { Self::Synth(SynthChallenges::new(challenges_per_partition)) } + pub const fn new_non_interactive(challenges_per_partition: usize) -> Self { + Self::Ni(NiChallenges::new(challenges_per_partition)) + } + pub fn num_challenges_per_partition(&self) -> usize { match self { Self::Interactive(InteractiveChallenges { @@ -141,6 +183,9 @@ impl Challenges { }) | Self::Synth(SynthChallenges { challenges_per_partition, + }) + | Self::Ni(NiChallenges { + challenges_per_partition, }) => *challenges_per_partition, } } diff --git a/storage-proofs-porep/src/stacked/vanilla/params.rs b/storage-proofs-porep/src/stacked/vanilla/params.rs index fcec5d63b..1fe410c05 100644 --- a/storage-proofs-porep/src/stacked/vanilla/params.rs +++ b/storage-proofs-porep/src/stacked/vanilla/params.rs @@ -188,6 +188,14 @@ impl PublicInputs { } } } + Challenges::Ni(ni_challenges) => { + let comm_r = self + .tau + .as_ref() + .expect("comm_r must be set prior to generating ni porep challenges") + .comm_r; + ni_challenges.derive(sector_nodes, &self.replica_id, &comm_r, k as u8) + } } } } diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index 25553beaf..947a67a77 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -109,11 +109,11 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr match challenges { Challenges::Interactive(interactive_challenges) => { - info!("generating non-synthetic vanilla proofs"); + info!("generating interactive vanilla proofs"); let seed = pub_inputs .seed - .expect("seed must be set for non-synthetic vanilla proofs"); + .expect("seed must be set for interactive vanilla proofs"); (0..partition_count) .map(|k| { @@ -191,6 +191,33 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr }) } } + Challenges::Ni(ni_challenges) => { + info!("generating non-interactive vanilla proofs"); + + let comm_r = pub_inputs.tau.as_ref().expect("tau is set").comm_r; + (0..partition_count) + .map(|k| { + trace!("proving partition {}/{}", k + 1, partition_count); + + // Derive the set of challenges we are proving over. + let challenge_positions = ni_challenges.derive( + graph.size(), + &pub_inputs.replica_id, + &comm_r, + k as u8, + ); + + Self::prove_layers_generate( + graph, + pub_inputs, + p_aux.comm_c, + t_aux, + challenge_positions, + num_layers, + ) + }) + .collect::>>>>() + } } }