From 1f3b2359c871fa5c422195bc800b500f6b6cd088 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 12 Oct 2023 20:07:42 +0200 Subject: [PATCH 1/3] feat: non-interactice PoRep Adding non-interactive PoRep. It's similar to the interactive PoRep, but it performance 12.6x the amount of proofs. In order to not overwhelm the system with too many parallel proofs, they are split into batches of 10, which then leads to a similar resource consumption per batch as for a single interactive PoRep. --- fil-proofs-tooling/src/shared.rs | 1 + filecoin-proofs/src/api/seal.rs | 86 ++++++++++++---- filecoin-proofs/src/constants.rs | 82 ++++++++-------- filecoin-proofs/src/parameters.rs | 19 ++-- filecoin-proofs/src/types/porep_config.rs | 53 ++++++++-- filecoin-proofs/tests/api.rs | 97 ++++++++++++++++--- storage-proofs-core/src/api_version.rs | 12 ++- storage-proofs-core/src/compound_proof.rs | 30 +++++- .../src/stacked/vanilla/challenges.rs | 45 +++++++++ .../src/stacked/vanilla/params.rs | 8 ++ .../src/stacked/vanilla/proof.rs | 31 +++++- 11 files changed, 367 insertions(+), 97 deletions(-) diff --git a/fil-proofs-tooling/src/shared.rs b/fil-proofs-tooling/src/shared.rs index 09b4c508e..2124f0940 100644 --- a/fil-proofs-tooling/src/shared.rs +++ b/fil-proofs-tooling/src/shared.rs @@ -339,4 +339,5 @@ pub fn get_porep_config( ) -> PoRepConfig { let arbitrary_porep_id = [99; 32]; PoRepConfig::new_groth16_with_features(sector_size, arbitrary_porep_id, api_version, features) + .expect("cannot set PoRep config") } diff --git a/filecoin-proofs/src/api/seal.rs b/filecoin-proofs/src/api/seal.rs index 246bb85bc..65824692f 100644 --- a/filecoin-proofs/src/api/seal.rs +++ b/filecoin-proofs/src/api/seal.rs @@ -33,7 +33,6 @@ use storage_proofs_porep::stacked::{ use storage_proofs_update::vanilla::prepare_tree_r_data; use typenum::{Unsigned, U11, U2}; -use crate::POREP_MINIMUM_CHALLENGES; use crate::{ api::util::{get_aggregate_target_len, pad_inputs_to_target, pad_proofs_to_target}, api::{as_safe_commitment, commitment_from_fr, get_base_tree_leafs, get_base_tree_size, util}, @@ -570,19 +569,50 @@ pub fn seal_commit_phase2( // Verification is cheap when parameters are cached, // and it is never correct to return a proof which does not verify. - verify_seal::( - porep_config, - comm_r, - comm_d, - prover_id, - sector_id, - ticket, - seed, - &buf, - ) - .context("post-seal verification sanity check failed")?; - - let out = SealCommitOutput { proof: buf }; + // Non-interactive PoRep is an aggregated proof, hence we also need a different code path for + // the verifucation. + let out = if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) { + let aggregated = aggregate_seal_commit_proofs::( + porep_config, + &[comm_r], + &[seed], + &[SealCommitOutput { proof: buf }], + groth16::aggregate::AggregateVersion::V2, + )?; + let inputs = get_seal_inputs::( + porep_config, + comm_r, + comm_d, + prover_id, + sector_id, + ticket, + seed, + )?; + let is_valid = verify_aggregate_seal_commit_proofs::( + porep_config, + aggregated.clone(), + &[comm_r], + &[seed], + inputs, + groth16::aggregate::AggregateVersion::V2, + ) + .context("post-seal aggregation verification sanity check failed")?; + ensure!(is_valid, "post seal aggregation verifies"); + SealCommitOutput { proof: aggregated } + } else { + verify_seal::( + porep_config, + comm_r, + comm_d, + prover_id, + sector_id, + ticket, + seed, + &buf, + ) + .context("post-seal verification sanity check failed")?; + SealCommitOutput { proof: buf } + }; info!("seal_commit_phase2:finish: {:?}", sector_id); Ok(out) @@ -892,6 +922,28 @@ pub fn verify_seal( ) -> Result { info!("verify_seal:start: {:?}", sector_id); + // Non-interactive PoReps are aggregated, but it should be possible to use the usual PoRep + // APIs, hence branch out here and not one layer higher. + if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) { + let inputs = get_seal_inputs::( + porep_config, + comm_r_in, + comm_d_in, + prover_id, + sector_id, + ticket, + seed, + )?; + return verify_aggregate_seal_commit_proofs::( + porep_config, + proof_vec.to_vec(), + &[comm_r_in], + &[seed], + inputs, + groth16::aggregate::AggregateVersion::V2, + ); + } + ensure!(comm_d_in != [0; 32], "Invalid all zero commitment (comm_d)"); ensure!(comm_r_in != [0; 32], "Invalid all zero commitment (comm_r)"); ensure!(!proof_vec.is_empty(), "Invalid proof bytes (empty vector)"); @@ -946,8 +998,7 @@ pub fn verify_seal( &public_inputs, &proof, &ChallengeRequirements { - minimum_challenges: POREP_MINIMUM_CHALLENGES - .from_sector_size(u64::from(porep_config.sector_size)), + minimum_challenges: porep_config.minimum_challenges(), }, ) }; @@ -1061,8 +1112,7 @@ pub fn verify_batch_seal( &public_inputs, &proofs, &ChallengeRequirements { - minimum_challenges: POREP_MINIMUM_CHALLENGES - .from_sector_size(u64::from(porep_config.sector_size)), + minimum_challenges: porep_config.minimum_challenges(), }, ) .map_err(Into::into); diff --git a/filecoin-proofs/src/constants.rs b/filecoin-proofs/src/constants.rs index 31e5b64ab..a96986089 100644 --- a/filecoin-proofs/src/constants.rs +++ b/filecoin-proofs/src/constants.rs @@ -1,5 +1,5 @@ +use std::collections::HashMap; use std::sync::RwLock; -use std::{collections::HashMap, sync::RwLockWriteGuard}; pub use storage_proofs_core::drgraph::BASE_DEGREE as DRG_DEGREE; pub use storage_proofs_porep::stacked::EXP_DEGREE; @@ -47,47 +47,7 @@ pub const PUBLISHED_SECTOR_SIZES: [u64; 10] = [ SECTOR_SIZE_64_GIB, ]; -pub struct PorepMinimumChallenges(RwLock>); -impl PorepMinimumChallenges { - fn new() -> Self { - Self(RwLock::new( - [ - (SECTOR_SIZE_2_KIB, 2), - (SECTOR_SIZE_4_KIB, 2), - (SECTOR_SIZE_16_KIB, 2), - (SECTOR_SIZE_32_KIB, 2), - (SECTOR_SIZE_8_MIB, 2), - (SECTOR_SIZE_16_MIB, 2), - (SECTOR_SIZE_512_MIB, 2), - (SECTOR_SIZE_1_GIB, 2), - (SECTOR_SIZE_32_GIB, 176), - (SECTOR_SIZE_64_GIB, 176), - ] - .iter() - .copied() - .collect(), - )) - } - - pub fn get_mut(&self) -> RwLockWriteGuard<'_, HashMap> { - self.0.write().expect("POREP_MINIMUM_CHALLENGES poisoned") - } - - pub fn from_sector_size(&self, sector_size: u64) -> usize { - match self - .0 - .read() - .expect("POREP_MINIMUM_CHALLENGES poisoned") - .get(§or_size) - { - Some(c) => *c, - None => panic!("invalid sector size"), - } - } -} - lazy_static! { - pub static ref POREP_MINIMUM_CHALLENGES: PorepMinimumChallenges = PorepMinimumChallenges::new(); pub static ref POREP_PARTITIONS: RwLock> = RwLock::new( [ (SECTOR_SIZE_2_KIB, 1), @@ -144,6 +104,43 @@ lazy_static! { ); } +/// Returns the minimum number of challenges used for the (synth and non-synth) interactive PoRep +/// for a certain sector size. +pub(crate) const fn get_porep_interactive_minimum_challenges(sector_size: u64) -> usize { + match sector_size { + SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB + | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 2, + SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 176, + _ => panic!("invalid sector size"), + } +} + +/// Returns the minimum number of challenges used for the non-interactive PoRep fo a certain sector +/// size. +pub(crate) const fn get_porep_non_interactive_minimum_challenges(sector_size: u64) -> usize { + match sector_size { + SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB + | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 4, + SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 2253, + _ => panic!("invalid sector size"), + } +} + +/// Returns the number of partitions for non-interactive PoRep for a certain sector size. +pub const fn get_porep_non_interactive_partitions(sector_size: u64) -> u8 { + match sector_size { + // The filename of the parameter files and verifying keys depend on the number of + // challenges per partition. In order to be able to re-use the files that were generated + // for the interactive PoRep, we need to use certain numbers, also for the test sector + // sizes. The number of challenges per partition for test sizes is 2, for production + // parameters it's 18. + SECTOR_SIZE_2_KIB | SECTOR_SIZE_4_KIB | SECTOR_SIZE_16_KIB | SECTOR_SIZE_32_KIB + | SECTOR_SIZE_8_MIB | SECTOR_SIZE_16_MIB | SECTOR_SIZE_512_MIB | SECTOR_SIZE_1_GIB => 2, + SECTOR_SIZE_32_GIB | SECTOR_SIZE_64_GIB => 126, + _ => panic!("invalid sector size"), + } +} + /// The size of a single snark proof. pub const SINGLE_PARTITION_PROOF_LEN: usize = 192; @@ -156,6 +153,9 @@ pub const MINIMUM_RESERVED_BYTES_FOR_PIECE_IN_FULLY_ALIGNED_SECTOR: u64 = /// The minimum size a single piece must have before padding. pub const MIN_PIECE_SIZE: UnpaddedBytesAmount = UnpaddedBytesAmount(127); +/// The maximum number of challenges per partition the circuits can work with. +pub(crate) const MAX_CHALLENGES_PER_PARTITION: u8 = 18; + /// The hasher used for creating comm_d. pub type DefaultPieceHasher = Sha256Hasher; pub type DefaultPieceDomain = ::Domain; diff --git a/filecoin-proofs/src/parameters.rs b/filecoin-proofs/src/parameters.rs index 4d97e98ad..01a2962e7 100644 --- a/filecoin-proofs/src/parameters.rs +++ b/filecoin-proofs/src/parameters.rs @@ -4,9 +4,8 @@ use storage_proofs_porep::stacked::{self, Challenges, StackedDrg}; use storage_proofs_post::fallback::{self, FallbackPoSt}; use crate::{ - constants::{DefaultPieceHasher, DRG_DEGREE, EXP_DEGREE, LAYERS}, + constants::{DefaultPieceHasher, DRG_DEGREE, EXP_DEGREE, LAYERS, MAX_CHALLENGES_PER_PARTITION}, types::{MerkleTreeTrait, PoRepConfig, PoStConfig}, - POREP_MINIMUM_CHALLENGES, }; type WinningPostSetupParams = fallback::SetupParams; @@ -68,12 +67,11 @@ pub fn window_post_setup_params(post_config: &PoStConfig) -> WindowPostSetupPara } pub fn setup_params(porep_config: &PoRepConfig) -> Result { - let use_synthetic = porep_config.feature_enabled(ApiFeature::SyntheticPoRep); let sector_bytes = porep_config.padded_bytes_amount(); let challenges = select_challenges( usize::from(porep_config.partitions), - POREP_MINIMUM_CHALLENGES.from_sector_size(u64::from(sector_bytes)), - use_synthetic, + porep_config.minimum_challenges(), + &porep_config.api_features, ); let num_layers = *LAYERS .read() @@ -113,11 +111,15 @@ const fn div_ceil(x: usize, y: usize) -> usize { fn select_challenges( partitions: usize, minimum_total_challenges: usize, - use_synthetic: bool, + features: &[ApiFeature], ) -> Challenges { let challenges = div_ceil(minimum_total_challenges, partitions); - if use_synthetic { + assert!(challenges <= usize::from(MAX_CHALLENGES_PER_PARTITION)); + + if features.contains(&ApiFeature::SyntheticPoRep) { Challenges::new_synthetic(challenges) + } else if features.contains(&ApiFeature::NonInteractivePoRep) { + Challenges::new_non_interactive(challenges) } else { Challenges::new_interactive(challenges) } @@ -131,8 +133,7 @@ mod tests { #[test] fn partition_layer_challenges_test() { - let f = - |partitions| select_challenges(partitions, 12, false).num_challenges_per_partition(); + let f = |partitions| select_challenges(partitions, 12, &[]).num_challenges_per_partition(); // Update to ensure all supported PoRepProofPartitions options are represented here. assert_eq!(6, f(usize::from(PoRepProofPartitions(2)))); diff --git a/filecoin-proofs/src/types/porep_config.rs b/filecoin-proofs/src/types/porep_config.rs index a923d95ed..e255e8387 100644 --- a/filecoin-proofs/src/types/porep_config.rs +++ b/filecoin-proofs/src/types/porep_config.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use anyhow::Result; +use anyhow::{anyhow, Result}; use storage_proofs_core::{ api_version::{ApiFeature, ApiVersion}, merkle::MerkleTreeTrait, @@ -12,7 +12,7 @@ use storage_proofs_core::{ use storage_proofs_porep::stacked::{StackedCircuit, StackedCompound}; use crate::{ - constants::DefaultPieceHasher, + constants::{self, DefaultPieceHasher}, parameters::public_params, types::{PaddedBytesAmount, PoRepProofPartitions, SectorSize, UnpaddedBytesAmount}, POREP_PARTITIONS, @@ -78,8 +78,8 @@ impl PoRepConfig { porep_id: [u8; 32], api_version: ApiVersion, api_features: Vec, - ) -> Self { - Self { + ) -> Result { + let mut config = Self { sector_size: SectorSize(sector_size), partitions: PoRepProofPartitions( *POREP_PARTITIONS @@ -90,15 +90,48 @@ impl PoRepConfig { ), porep_id, api_version, - api_features, + api_features: vec![], + }; + for feature in api_features { + config.enable_feature(feature)?; } + Ok(config) } #[inline] - pub fn enable_feature(&mut self, feat: ApiFeature) { + pub fn enable_feature(&mut self, feat: ApiFeature) -> Result<()> { + match feat { + ApiFeature::SyntheticPoRep => { + if self.feature_enabled(ApiFeature::NonInteractivePoRep) { + return Err(anyhow!( + "Cannot enable Synthetic PoRep when Non-interactive PoRep is already enabled")); + } + + self.partitions = PoRepProofPartitions( + *POREP_PARTITIONS + .read() + .expect("POREP_PARTITIONS poisoned") + .get(&self.sector_size.into()) + .expect("unknown sector size"), + ); + } + ApiFeature::NonInteractivePoRep => { + if self.feature_enabled(ApiFeature::SyntheticPoRep) { + return Err(anyhow!( + "Cannot enable Non-interactive PoRep when Synthetic PoRep is already enabled")); + } + + self.partitions = PoRepProofPartitions( + constants::get_porep_non_interactive_partitions(self.sector_size.into()), + ); + } + } + if !self.feature_enabled(feat) { self.api_features.push(feat); } + + Ok(()) } #[inline] @@ -116,6 +149,14 @@ impl PoRepConfig { self.padded_bytes_amount().into() } + pub fn minimum_challenges(&self) -> usize { + if self.feature_enabled(ApiFeature::NonInteractivePoRep) { + constants::get_porep_non_interactive_minimum_challenges(u64::from(self.sector_size)) + } else { + constants::get_porep_interactive_minimum_challenges(u64::from(self.sector_size)) + } + } + /// Returns the cache identifier as used by `storage-proofs::parameter_cache`. pub fn get_cache_identifier(&self) -> Result { let params = public_params::(self)?; diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index b8e624fb3..cc024fc4b 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -11,8 +11,8 @@ use ff::Field; use filecoin_hashers::Hasher; use filecoin_proofs::{ add_piece, aggregate_empty_sector_update_proofs, aggregate_seal_commit_proofs, clear_cache, - clear_layer_data, clear_synthetic_proofs, compute_comm_d, decode_from, decode_from_range, - encode_into, fauxrep_aux, generate_empty_sector_update_proof, + clear_synthetic_proofs, compute_comm_d, decode_from, decode_from_range, encode_into, + fauxrep_aux, generate_empty_sector_update_proof, generate_empty_sector_update_proof_with_vanilla, generate_fallback_sector_challenges, generate_partition_proofs, generate_piece_commitment, generate_single_partition_proof, generate_single_vanilla_proof, generate_single_window_post_with_vanilla, generate_synth_proofs, @@ -174,7 +174,7 @@ fn test_seal_lifecycle_2kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -204,6 +204,11 @@ fn test_seal_lifecycle_upgrade_2kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + MAX_LEGACY_REGISTERED_SEAL_PROOF_ID + 1, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id_num, api_version, features) in test_inputs { @@ -213,7 +218,7 @@ fn test_seal_lifecycle_upgrade_2kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle_upgrade::(&porep_config)?; } @@ -233,6 +238,11 @@ fn test_seal_lifecycle_4kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -241,7 +251,7 @@ fn test_seal_lifecycle_4kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -261,6 +271,11 @@ fn test_seal_lifecycle_upgrade_4kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -269,7 +284,7 @@ fn test_seal_lifecycle_upgrade_4kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle_upgrade::(&porep_config)?; } @@ -289,6 +304,11 @@ fn test_seal_lifecycle_16kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -297,7 +317,7 @@ fn test_seal_lifecycle_16kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -317,6 +337,11 @@ fn test_seal_lifecycle_upgrade_16kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -325,7 +350,7 @@ fn test_seal_lifecycle_upgrade_16kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle_upgrade::(&porep_config)?; } @@ -345,6 +370,11 @@ fn test_seal_lifecycle_32kib_base_8() -> Result<()> { ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], ), + ( + ARBITRARY_POREP_ID_V1_2_0, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + ), ]; for (porep_id, api_version, features) in test_inputs { @@ -353,7 +383,7 @@ fn test_seal_lifecycle_32kib_base_8() -> Result<()> { porep_id, api_version, features, - ); + )?; seal_lifecycle::(&porep_config)?; } @@ -436,9 +466,30 @@ fn test_seal_lifecycle_32gib_porep_id_v1_2_top_8_8_0_api_v1_2() -> Result<()> { let porep_id = to_porep_id_verified(porep_id_v1_2, ApiVersion::V1_2_0); assert!(!is_legacy_porep_id(porep_id)); - let mut porep_config = - PoRepConfig::new_groth16(SECTOR_SIZE_32_GIB, porep_id, ApiVersion::V1_2_0); - porep_config.enable_feature(ApiFeature::SyntheticPoRep); + let porep_config = PoRepConfig::new_groth16_with_features( + SECTOR_SIZE_32_GIB, + porep_id, + ApiVersion::V1_2_0, + vec![ApiFeature::SyntheticPoRep], + )?; + + seal_lifecycle::(&porep_config) +} + +#[cfg(feature = "big-tests")] +#[test] +fn test_seal_lifecycle_32gib_porep_id_v1_2_ni_top_8_8_0_api_v1_2() -> Result<()> { + let porep_id_v1_2: u64 = 8; // This is a RegisteredSealProof value + + let porep_id = to_porep_id_verified(porep_id_v1_2, ApiVersion::V1_2_0); + assert!(!is_legacy_porep_id(porep_id)); + + let porep_config = PoRepConfig::new_groth16_with_features( + SECTOR_SIZE_32_GIB, + porep_id, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + )?; seal_lifecycle::(&porep_config) } @@ -493,7 +544,25 @@ fn test_seal_lifecycle_64gib_porep_id_v1_2_top_8_8_2_api_v1_2() -> Result<()> { porep_id, ApiVersion::V1_2_0, vec![ApiFeature::SyntheticPoRep], - ); + )?; + + seal_lifecycle::(&porep_config) +} + +#[cfg(feature = "big-tests")] +#[test] +fn test_seal_lifecycle_64gib_porep_id_v1_2_ni_top_8_8_2_api_v1_2() -> Result<()> { + let porep_id_v1_2: u64 = 9; // This is a RegisteredSealProof value + + let porep_id = to_porep_id_verified(porep_id_v1_2, ApiVersion::V1_2_0); + assert!(!is_legacy_porep_id(porep_id)); + + let porep_config = PoRepConfig::new_groth16_with_features( + SECTOR_SIZE_64_GIB, + porep_id, + ApiVersion::V1_2_0, + vec![ApiFeature::NonInteractivePoRep], + )?; seal_lifecycle::(&porep_config) } @@ -2570,7 +2639,7 @@ fn create_seal_for_upgrade_aggregation< pre_commit_output, &piece_infos, )?; - clear_layer_data::(cache_dir.path())?; + clear_cache::(cache_dir.path())?; } else { info!("SyntheticPoRep is NOT enabled"); validate_cache_for_commit::<_, _, Tree>(cache_dir.path(), sealed_sector_file.path())?; diff --git a/storage-proofs-core/src/api_version.rs b/storage-proofs-core/src/api_version.rs index 5576b3d67..8dacfd745 100644 --- a/storage-proofs-core/src/api_version.rs +++ b/storage-proofs-core/src/api_version.rs @@ -95,20 +95,21 @@ impl FromStr for ApiVersion { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ApiFeature { SyntheticPoRep, + NonInteractivePoRep, } impl ApiFeature { #[inline] pub fn first_supported_version(&self) -> ApiVersion { match self { - ApiFeature::SyntheticPoRep => ApiVersion::V1_2_0, + ApiFeature::SyntheticPoRep | ApiFeature::NonInteractivePoRep => ApiVersion::V1_2_0, } } #[inline] pub fn last_supported_version(&self) -> Option { match self { - ApiFeature::SyntheticPoRep => None, + ApiFeature::SyntheticPoRep | ApiFeature::NonInteractivePoRep => None, } } } @@ -145,3 +146,10 @@ fn test_api_feature_synthetic_porep() { assert!(feature.first_supported_version() == ApiVersion::V1_2_0); assert!(feature.last_supported_version().is_none()); } + +#[test] +fn test_api_feature_non_interactive_porep() { + let feature = ApiFeature::NonInteractivePoRep; + assert!(feature.first_supported_version() == ApiVersion::V1_2_0); + assert!(feature.last_supported_version().is_none()); +} diff --git a/storage-proofs-core/src/compound_proof.rs b/storage-proofs-core/src/compound_proof.rs index ccc18e455..7ba81fce4 100644 --- a/storage-proofs-core/src/compound_proof.rs +++ b/storage-proofs-core/src/compound_proof.rs @@ -1,3 +1,5 @@ +use std::cmp; + use anyhow::{ensure, Context}; use bellperson::{ groth16::{ @@ -25,6 +27,12 @@ use crate::{ proof::ProofScheme, }; +/// The maximum number of Groth16 proofs that will be processed in parallel. This limit is set as +/// synthesis takes a lot of memory. The current value is based on the number of proofs that are +/// run in parallel in the interactive PoRep (the number of partitions). This way there's just a +/// single batch for the interactive PoRep, but the non-interactive PoRep is split into batches. +const MAX_GROTH16_BATCH_SIZE: usize = 10; + #[derive(Clone)] pub struct SetupParams<'a, S: ProofScheme<'a>> { pub vanilla_params: >::SetupParams, @@ -242,7 +250,7 @@ where "cannot create a circuit proof over missing vanilla proofs" ); - let circuits = vanilla_proofs + let mut circuits = vanilla_proofs .into_par_iter() .enumerate() .map(|(k, vanilla_proof)| { @@ -256,14 +264,26 @@ where }) .collect::>>()?; - let groth_proofs = if priority { - create_random_proof_batch_in_priority(circuits, groth_params, &mut rng)? + // The same function is used within the while loop below, decide on the function once and + // not on every iteration. + let create_random_proof_batch_fun = if priority { + create_random_proof_batch_in_priority } else { - create_random_proof_batch(circuits, groth_params, &mut rng)? + create_random_proof_batch }; + let mut groth_proofs = Vec::with_capacity(circuits.len()); + // Bellperson expects a vector of proofs, hence drain it from the list of proofs, so that + // we don't need to keep an extra copy around. + while !circuits.is_empty() { + let size = cmp::min(MAX_GROTH16_BATCH_SIZE, circuits.len()); + let batch = circuits.drain(0..size).collect(); + let proofs = create_random_proof_batch_fun(batch, groth_params, &mut rng)?; + groth_proofs.extend_from_slice(&proofs); + } + groth_proofs - .into_iter() + .iter() .map(|groth_proof| { let mut proof_vec = Vec::new(); groth_proof.write(&mut proof_vec)?; diff --git a/storage-proofs-porep/src/stacked/vanilla/challenges.rs b/storage-proofs-porep/src/stacked/vanilla/challenges.rs index e520af508..9214a79c5 100644 --- a/storage-proofs-porep/src/stacked/vanilla/challenges.rs +++ b/storage-proofs-porep/src/stacked/vanilla/challenges.rs @@ -119,10 +119,48 @@ impl SynthChallenges { } } +#[derive(Clone, Debug)] +pub struct NiChallenges { + challenges_per_partition: usize, +} + +impl NiChallenges { + pub const fn new(challenges_per_partition: usize) -> Self { + Self { + challenges_per_partition, + } + } + + pub fn derive( + &self, + sector_nodes: usize, + replica_id: &D, + comm_r: &D, + k: u8, + ) -> Vec { + const TAG: &[u8] = b"filecoin.io|PoRep|1|NonInteractive|1"; + let hash_init = Sha256::new() + .chain_update(TAG) + .chain_update(replica_id.into_bytes()) + .chain_update(comm_r); + (0..self.challenges_per_partition) + .map(|i| { + let j: u32 = ((self.challenges_per_partition * k as usize) + i) as u32; + + let hash = hash_init.clone().chain_update(j.to_le_bytes()).finalize(); + + let bigint = BigUint::from_bytes_le(hash.as_ref()); + bigint_to_challenge(bigint, sector_nodes) + }) + .collect() + } +} + #[derive(Clone, Debug)] pub enum Challenges { Interactive(InteractiveChallenges), Synth(SynthChallenges), + Ni(NiChallenges), } impl Challenges { @@ -134,6 +172,10 @@ impl Challenges { Self::Synth(SynthChallenges::new(challenges_per_partition)) } + pub const fn new_non_interactive(challenges_per_partition: usize) -> Self { + Self::Ni(NiChallenges::new(challenges_per_partition)) + } + pub fn num_challenges_per_partition(&self) -> usize { match self { Self::Interactive(InteractiveChallenges { @@ -141,6 +183,9 @@ impl Challenges { }) | Self::Synth(SynthChallenges { challenges_per_partition, + }) + | Self::Ni(NiChallenges { + challenges_per_partition, }) => *challenges_per_partition, } } diff --git a/storage-proofs-porep/src/stacked/vanilla/params.rs b/storage-proofs-porep/src/stacked/vanilla/params.rs index fcec5d63b..1fe410c05 100644 --- a/storage-proofs-porep/src/stacked/vanilla/params.rs +++ b/storage-proofs-porep/src/stacked/vanilla/params.rs @@ -188,6 +188,14 @@ impl PublicInputs { } } } + Challenges::Ni(ni_challenges) => { + let comm_r = self + .tau + .as_ref() + .expect("comm_r must be set prior to generating ni porep challenges") + .comm_r; + ni_challenges.derive(sector_nodes, &self.replica_id, &comm_r, k as u8) + } } } } diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index 25553beaf..947a67a77 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -109,11 +109,11 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr match challenges { Challenges::Interactive(interactive_challenges) => { - info!("generating non-synthetic vanilla proofs"); + info!("generating interactive vanilla proofs"); let seed = pub_inputs .seed - .expect("seed must be set for non-synthetic vanilla proofs"); + .expect("seed must be set for interactive vanilla proofs"); (0..partition_count) .map(|k| { @@ -191,6 +191,33 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr }) } } + Challenges::Ni(ni_challenges) => { + info!("generating non-interactive vanilla proofs"); + + let comm_r = pub_inputs.tau.as_ref().expect("tau is set").comm_r; + (0..partition_count) + .map(|k| { + trace!("proving partition {}/{}", k + 1, partition_count); + + // Derive the set of challenges we are proving over. + let challenge_positions = ni_challenges.derive( + graph.size(), + &pub_inputs.replica_id, + &comm_r, + k as u8, + ); + + Self::prove_layers_generate( + graph, + pub_inputs, + p_aux.comm_c, + t_aux, + challenge_positions, + num_layers, + ) + }) + .collect::>>>>() + } } } From e2153ad47ecfcbe37756ad8a736083fa700a4952 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Tue, 19 Dec 2023 20:27:53 +0100 Subject: [PATCH 2/3] refactor: define conflicting features in a more general way --- filecoin-proofs/src/types/porep_config.rs | 18 ++++++++---------- storage-proofs-core/src/api_version.rs | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/filecoin-proofs/src/types/porep_config.rs b/filecoin-proofs/src/types/porep_config.rs index e255e8387..2eff1b324 100644 --- a/filecoin-proofs/src/types/porep_config.rs +++ b/filecoin-proofs/src/types/porep_config.rs @@ -100,13 +100,16 @@ impl PoRepConfig { #[inline] pub fn enable_feature(&mut self, feat: ApiFeature) -> Result<()> { + for conflict in feat.conflicting_features() { + if self.feature_enabled(*conflict) { + return Err(anyhow!( + "Cannot enable feature `{feat}` when `{conflict}` is already enabled" + )); + } + } + match feat { ApiFeature::SyntheticPoRep => { - if self.feature_enabled(ApiFeature::NonInteractivePoRep) { - return Err(anyhow!( - "Cannot enable Synthetic PoRep when Non-interactive PoRep is already enabled")); - } - self.partitions = PoRepProofPartitions( *POREP_PARTITIONS .read() @@ -116,11 +119,6 @@ impl PoRepConfig { ); } ApiFeature::NonInteractivePoRep => { - if self.feature_enabled(ApiFeature::SyntheticPoRep) { - return Err(anyhow!( - "Cannot enable Non-interactive PoRep when Synthetic PoRep is already enabled")); - } - self.partitions = PoRepProofPartitions( constants::get_porep_non_interactive_partitions(self.sector_size.into()), ); diff --git a/storage-proofs-core/src/api_version.rs b/storage-proofs-core/src/api_version.rs index 8dacfd745..a0966b54f 100644 --- a/storage-proofs-core/src/api_version.rs +++ b/storage-proofs-core/src/api_version.rs @@ -112,6 +112,24 @@ impl ApiFeature { ApiFeature::SyntheticPoRep | ApiFeature::NonInteractivePoRep => None, } } + + /// Return the features that are in conflict with the current one. + pub fn conflicting_features(&self) -> &[ApiFeature] { + match self { + ApiFeature::SyntheticPoRep => &[ApiFeature::NonInteractivePoRep], + ApiFeature::NonInteractivePoRep => &[ApiFeature::SyntheticPoRep], + } + } +} + +impl Display for ApiFeature { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let api_feature_str = match self { + Self::SyntheticPoRep => "synthetic-porep", + Self::NonInteractivePoRep => "non-interactive-porep", + }; + write!(f, "{}", api_feature_str) + } } #[test] From 23c09732dd56c384078b76c3ba36be69f5118e07 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Tue, 19 Dec 2023 20:46:41 +0100 Subject: [PATCH 3/3] refactor: simplify the verification logic --- filecoin-proofs/src/api/seal.rs | 61 +++++++++++---------------------- 1 file changed, 20 insertions(+), 41 deletions(-) diff --git a/filecoin-proofs/src/api/seal.rs b/filecoin-proofs/src/api/seal.rs index 65824692f..d77204680 100644 --- a/filecoin-proofs/src/api/seal.rs +++ b/filecoin-proofs/src/api/seal.rs @@ -567,55 +567,34 @@ pub fn seal_commit_phase2( proof.write(&mut buf)?; - // Verification is cheap when parameters are cached, - // and it is never correct to return a proof which does not verify. - // Non-interactive PoRep is an aggregated proof, hence we also need a different code path for - // the verifucation. - let out = if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) { - let aggregated = aggregate_seal_commit_proofs::( + // Non-interactive PoRep is an aggregated proof, hence we use that as the returned buffer. + if porep_config.feature_enabled(ApiFeature::NonInteractivePoRep) { + buf = aggregate_seal_commit_proofs::( porep_config, &[comm_r], &[seed], &[SealCommitOutput { proof: buf }], groth16::aggregate::AggregateVersion::V2, )?; - let inputs = get_seal_inputs::( - porep_config, - comm_r, - comm_d, - prover_id, - sector_id, - ticket, - seed, - )?; - let is_valid = verify_aggregate_seal_commit_proofs::( - porep_config, - aggregated.clone(), - &[comm_r], - &[seed], - inputs, - groth16::aggregate::AggregateVersion::V2, - ) - .context("post-seal aggregation verification sanity check failed")?; - ensure!(is_valid, "post seal aggregation verifies"); - SealCommitOutput { proof: aggregated } - } else { - verify_seal::( - porep_config, - comm_r, - comm_d, - prover_id, - sector_id, - ticket, - seed, - &buf, - ) - .context("post-seal verification sanity check failed")?; - SealCommitOutput { proof: buf } - }; + } + + // Verification is cheap when parameters are cached, + // and it is never correct to return a proof which does not verify. + let is_valid = verify_seal::( + porep_config, + comm_r, + comm_d, + prover_id, + sector_id, + ticket, + seed, + &buf, + ) + .context("post-seal verification sanity check failed")?; + ensure!(is_valid, "post seal aggregation verifies"); info!("seal_commit_phase2:finish: {:?}", sector_id); - Ok(out) + Ok(SealCommitOutput { proof: buf }) } /// Given the specified arguments, this method returns the inputs that were used to