diff --git a/filecoin-proofs/src/api/post_util.rs b/filecoin-proofs/src/api/post_util.rs index 1c76fd96e5..27fe24bb5e 100644 --- a/filecoin-proofs/src/api/post_util.rs +++ b/filecoin-proofs/src/api/post_util.rs @@ -7,7 +7,8 @@ use bincode::deserialize; use filecoin_hashers::Hasher; use log::{info, trace}; use storage_proofs_core::{ - cache_key::CacheKey, merkle::MerkleTreeTrait, proof::ProofScheme, sector::SectorId, + cache_key::CacheKey, compound_proof::PubVanRsp, merkle::MerkleTreeTrait, proof::ProofScheme, + sector::SectorId, }; use storage_proofs_post::fallback::{self, generate_leaf_challenge, FallbackPoSt, SectorProof}; @@ -308,3 +309,139 @@ pub(crate) fn get_partitions_for_window_post( None } } + +pub fn get_post_vanilla_params( + post_config: &PoStConfig, +) -> Result { + let rsp = PubVanRsp { + challengecount: post_config.challenge_count, + sectorcount: post_config.sector_count, + }; + + Ok(rsp) +} + +pub fn single_partition_vanilla_proofs( + post_config: &PoStConfig, + pub_params: &fallback::PublicParams, + pub_inputs: &fallback::PublicInputs<::Domain>, + partition_count: usize, + vanilla_proofs: &[FallbackPoStSectorProof], + sector_idxs: &[u64], +) -> Result>> { + info!("single_partition_vanilla_proofs:start"); + ensure!( + post_config.typ == PoStType::Window || post_config.typ == PoStType::Winning, + "invalid post config type" + ); + + let num_sectors_per_chunk = pub_params.sector_count; + let num_sectors = pub_inputs.sectors.len(); + + ensure!( + num_sectors <= partition_count * num_sectors_per_chunk, + "cannot prove the provided number of sectors: {} > {} * {}", + num_sectors, + partition_count, + num_sectors_per_chunk, + ); + + let mut partition_proofs = Vec::new(); + + // Note that the partition proofs returned are shaped differently + // based on which type of PoSt is being considered. + match post_config.typ { + PoStType::Window => { + for (j, sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + let mut sector_proofs = Vec::with_capacity(num_sectors_per_chunk); + + for pub_sector in sectors_chunk.iter() { + let cur_proof = vanilla_proofs + .iter() + .find(|&proof| proof.sector_id == pub_sector.id) + .expect("failed to locate sector proof"); + + // Note: Window post requires all inclusion proofs (based on the challenge + // count per sector) per sector proof. + sector_proofs.extend(cur_proof.vanilla_proof.sectors.clone()); + } + + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while sector_proofs.len() < num_sectors_per_chunk { + sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone()); + } + + partition_proofs.push(fallback::Proof::<::Proof> { + sectors: sector_proofs, + }); + } + } + PoStType::Winning => { + for (j, sectors_chunk) in vanilla_proofs.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + // Sanity check incoming structure + ensure!( + sectors_chunk.len() == 1, + "Invalid sector chunk for Winning PoSt" + ); + ensure!( + sectors_chunk[0].vanilla_proof.sectors.len() == 1, + "Invalid sector count for Winning PoSt chunk" + ); + + // Winning post sector_count is winning post challenges per sector + ensure!( + post_config.sector_count == sectors_chunk[j].vanilla_proof.sectors.len(), + "invalid number of sector proofs for Winning PoSt" + ); + + let mut sector_proofs = Vec::with_capacity(post_config.challenge_count); + let cur_sector_proof = §ors_chunk[0].vanilla_proof.sectors[0]; + + // Unroll inclusions proofs from the single provided sector_proof (per partition) + // into individual sector proofs, required for winning post. + for cur_inclusion_proof in cur_sector_proof.inclusion_proofs() { + sector_proofs.push(SectorProof { + inclusion_proofs: vec![cur_inclusion_proof.clone()], + comm_c: cur_sector_proof.comm_c, + comm_r_last: cur_sector_proof.comm_r_last, + }); + } + + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while sector_proofs.len() < num_sectors_per_chunk { + sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone()); + } + + // Winning post Challenge count is the total winning post challenges + ensure!( + sector_proofs.len() == post_config.challenge_count, + "invalid number of partition proofs based on Winning PoSt challenges" + ); + + partition_proofs.push(fallback::Proof::<::Proof> { + sectors: sector_proofs, + }); + } + } + } + + info!("single_partition_vanilla_proofs:finish"); + + ensure!( + FallbackPoSt::::verify_single_partitions( + pub_params, + pub_inputs, + &partition_proofs, + sector_idxs + )?, + "partitioned vanilla proofs failed to verify" + ); + + Ok(partition_proofs) +} diff --git a/filecoin-proofs/src/api/window_post.rs b/filecoin-proofs/src/api/window_post.rs index 99c1356ed5..ab4ba49671 100644 --- a/filecoin-proofs/src/api/window_post.rs +++ b/filecoin-proofs/src/api/window_post.rs @@ -14,7 +14,10 @@ use storage_proofs_post::fallback::{ }; use crate::{ - api::{as_safe_commitment, get_partitions_for_window_post, partition_vanilla_proofs}, + api::{ + as_safe_commitment, get_partitions_for_window_post, partition_vanilla_proofs, + single_partition_vanilla_proofs, + }, caches::{get_post_params, get_post_verifying_key}, parameters::window_post_setup_params, types::{ @@ -242,3 +245,73 @@ pub fn verify_window_post( Ok(true) } + +/// Generates a Window proof-of-spacetime with provided vanilla proofs of a single partition +pub fn generate_single_window_post_with_vanilla( + post_config: &PoStConfig, + randomness: &ChallengeSeed, + prover_id: ProverId, + vanilla_proofs: Vec>, + sector_idxs: &[u64], +) -> Result { + info!("generate_single_window_post_with_vanilla:start"); + ensure!( + post_config.typ == PoStType::Window, + "invalid post config type" + ); + + let randomness_safe: ::Domain = + as_safe_commitment(randomness, "randomness")?; + let prover_id_safe: ::Domain = + as_safe_commitment(&prover_id, "prover_id")?; + + let vanilla_params = window_post_setup_params(&post_config); + let partitions = get_partitions_for_window_post(vanilla_proofs.len(), &post_config); + + let setup_params = compound_proof::SetupParams { + vanilla_params, + partitions, + priority: post_config.priority, + }; + + let partitions = partitions.unwrap_or(1); + + let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, Tree>> = + FallbackPoStCompound::setup(&setup_params)?; + let groth_params = get_post_params::(&post_config)?; + + let mut pub_sectors = Vec::with_capacity(vanilla_proofs.len()); + for vanilla_proof in &vanilla_proofs { + pub_sectors.push(PublicSector { + id: vanilla_proof.sector_id, + comm_r: vanilla_proof.comm_r, + }); + } + + let pub_inputs = fallback::PublicInputs { + randomness: randomness_safe, + prover_id: prover_id_safe, + sectors: pub_sectors, + k: None, + }; + + let partitioned_proofs = single_partition_vanilla_proofs( + &post_config, + &pub_params.vanilla_params, + &pub_inputs, + partitions, + &vanilla_proofs, + sector_idxs, + )?; + + let proof = FallbackPoStCompound::prove_with_vanilla( + &pub_params, + &pub_inputs, + partitioned_proofs, + &groth_params, + )?; + + info!("generate_single_window_post_with_vanilla:finish"); + + proof.to_vec() +} diff --git a/filecoin-proofs/tests/api.rs b/filecoin-proofs/tests/api.rs index b972626e8f..49ca44cdea 100644 --- a/filecoin-proofs/tests/api.rs +++ b/filecoin-proofs/tests/api.rs @@ -13,7 +13,8 @@ use filecoin_hashers::Hasher; use filecoin_proofs::{ add_piece, aggregate_seal_commit_proofs, clear_cache, compute_comm_d, fauxrep_aux, generate_fallback_sector_challenges, generate_piece_commitment, generate_single_vanilla_proof, - generate_window_post, generate_window_post_with_vanilla, generate_winning_post, + generate_single_window_post_with_vanilla, generate_window_post, + generate_window_post_with_vanilla, generate_winning_post, generate_winning_post_sector_challenge, generate_winning_post_with_vanilla, get_seal_inputs, seal_commit_phase1, seal_commit_phase2, seal_pre_commit_phase1, seal_pre_commit_phase2, unseal_range, validate_cache_for_commit, validate_cache_for_precommit_phase2, @@ -1055,6 +1056,182 @@ fn test_window_post_single_partition_matching_2kib_base_8() -> Result<()> { ) } +#[test] +fn test_window_post_partition_matching_2kib_base_8() -> Result<()> { + let sector_size = SECTOR_SIZE_2_KIB; + let sector_count = *WINDOW_POST_SECTOR_COUNT + .read() + .expect("WINDOW_POST_SECTOR_COUNT poisoned") + .get(§or_size) + .expect("unknown sector size"); + + partition_window_post::( + sector_size, + 3, // Validate the scenarios of two partition + sector_count, + false, + ApiVersion::V1_0_0, + )?; + partition_window_post::( + sector_size, + 3, + sector_count, + true, + ApiVersion::V1_0_0, + )?; + partition_window_post::( + sector_size, + 3, + sector_count, + false, + ApiVersion::V1_1_0, + )?; + partition_window_post::(sector_size, 3, sector_count, true, ApiVersion::V1_1_0) +} + +fn partition_window_post( + sector_size: u64, + total_sector_count: usize, + sector_count: usize, + fake: bool, + api_version: ApiVersion, +) -> Result<()> { + use anyhow::anyhow; + + let rng = &mut XorShiftRng::from_seed(TEST_SEED); + + let mut sectors = Vec::with_capacity(total_sector_count); + let mut pub_replicas = BTreeMap::new(); + let mut priv_replicas = BTreeMap::new(); + + let prover_fr: ::Domain = Fr::random(rng).into(); + let mut prover_id = [0u8; 32]; + prover_id.copy_from_slice(AsRef::<[u8]>::as_ref(&prover_fr)); + + let porep_id = match api_version { + ApiVersion::V1_0_0 => ARBITRARY_POREP_ID_V1_0_0, + ApiVersion::V1_1_0 => ARBITRARY_POREP_ID_V1_1_0, + }; + + for _ in 0..total_sector_count { + let (sector_id, replica, comm_r, cache_dir) = if fake { + create_fake_seal::<_, Tree>(rng, sector_size, &porep_id, api_version)? + } else { + create_seal::<_, Tree>(rng, sector_size, prover_id, true, &porep_id, api_version)? + }; + priv_replicas.insert( + sector_id, + PrivateReplicaInfo::new(replica.path().into(), comm_r, cache_dir.path().into())?, + ); + pub_replicas.insert(sector_id, PublicReplicaInfo::new(comm_r)?); + sectors.push((sector_id, replica, comm_r, cache_dir, prover_id)); + } + assert_eq!(priv_replicas.len(), total_sector_count); + assert_eq!(pub_replicas.len(), total_sector_count); + assert_eq!(sectors.len(), total_sector_count); + + let random_fr: ::Domain = Fr::random(rng).into(); + let mut randomness = [0u8; 32]; + randomness.copy_from_slice(AsRef::<[u8]>::as_ref(&random_fr)); + + let config = PoStConfig { + sector_size: sector_size.into(), + sector_count, + challenge_count: WINDOW_POST_CHALLENGE_COUNT, + typ: PoStType::Window, + priority: false, + api_version, + }; + + let replica_sectors = priv_replicas + .iter() + .map(|(sector, _replica)| *sector) + .collect::>(); + + let challenges = generate_fallback_sector_challenges::( + &config, + &randomness, + &replica_sectors, + prover_id, + )?; + + let mut start = 0; + let mut _end = config.sector_count; + let num_sectors_per_chunk = config.sector_count; + let mut proofs = Vec::new(); + + let partitions = match config.typ { + PoStType::Window => { + let partitions = + ((replica_sectors.len()) as f32 / config.sector_count as f32).ceil() as usize; + if partitions > 1 { + partitions + } else { + 1 + } + } + PoStType::Winning => 1, + }; + for partition_index in 0..partitions { + let sector_ids = replica_sectors + .chunks(num_sectors_per_chunk) + .nth(partition_index) + .ok_or_else(|| anyhow!("invalid number of sectors/partition index"))?; + + let mut partition_priv_replicas = BTreeMap::new(); + for id in sector_ids { + let p_sector = match priv_replicas.get(id) { + Some(v) => v, + _ => { + continue; + } + }; + + partition_priv_replicas.insert(*id, p_sector); + } + + let mut sector_idxs = Vec::new(); + let mut vanilla_proofs = Vec::new(); + for (i, (sector_id, sector)) in partition_priv_replicas.iter().enumerate() { + let sector_index = i + start; + sector_idxs.push(sector_index as u64); + + let sector_challenges = &challenges[sector_id]; + let single_proof = generate_single_vanilla_proof::( + &config, + *sector_id, + sector, + sector_challenges, + )?; + + vanilla_proofs.push(single_proof); + } + + let mut proof = generate_single_window_post_with_vanilla( + &config, + &randomness, + prover_id, + vanilla_proofs, + §or_idxs, + )?; + + if partition_index == 0 { + proofs = proof.clone(); + } else { + proofs.append(&mut proof); + } + + start = start + config.sector_count; + _end = _end + config.sector_count; + } + + let valid = + verify_window_post::(&config, &randomness, &pub_replicas, prover_id, &proofs)?; + assert!(valid, "proofs did not verify"); + + Ok(()) +} + fn window_post( sector_size: u64, total_sector_count: usize, diff --git a/storage-proofs-core/src/compound_proof.rs b/storage-proofs-core/src/compound_proof.rs index 6532a835fb..4823a74360 100644 --- a/storage-proofs-core/src/compound_proof.rs +++ b/storage-proofs-core/src/compound_proof.rs @@ -16,6 +16,7 @@ use rand::{rngs::OsRng, RngCore}; use rayon::prelude::{ IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, }; +use serde::{Deserialize, Serialize}; use crate::{ error::Result, @@ -497,3 +498,9 @@ where Ok(res) } } + +#[derive(Serialize, Deserialize)] +pub struct PubVanRsp { + pub challengecount: usize, + pub sectorcount: usize, +} diff --git a/storage-proofs-core/src/proof.rs b/storage-proofs-core/src/proof.rs index 2b39a9fc69..9f6320cde9 100644 --- a/storage-proofs-core/src/proof.rs +++ b/storage-proofs-core/src/proof.rs @@ -85,6 +85,23 @@ pub trait ProofScheme<'a> { Ok(true) } + fn verify_single_partitions( + pub_params: &Self::PublicParams, + pub_in: &Self::PublicInputs, + proofs: &[Self::Proof], + _sector_idxs: &[u64], + ) -> Result { + for (k, proof) in proofs.iter().enumerate() { + let partition_pub_in = Self::with_partition((*pub_in).clone(), Some(k)); // + + if !Self::verify(pub_params, &partition_pub_in, proof)? { + return Ok(false); + } + } + + Ok(true) + } + // This method must be specialized by concrete ProofScheme implementations which use partitions. fn with_partition(pub_in: Self::PublicInputs, _k: Option) -> Self::PublicInputs { pub_in diff --git a/storage-proofs-post/src/fallback/vanilla.rs b/storage-proofs-post/src/fallback/vanilla.rs index ac7441b7a6..cbce1d8312 100644 --- a/storage-proofs-post/src/fallback/vanilla.rs +++ b/storage-proofs-post/src/fallback/vanilla.rs @@ -630,4 +630,134 @@ impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for FallbackPoSt<'a, Tree> checked * public_params.challenge_count >= requirements.minimum_challenge_count } + + // Verify vanilla proofs for a single partition. + // Calculate the offset for the challenge position of the sector + fn verify_single_partitions( + pub_params: &Self::PublicParams, + pub_inputs: &Self::PublicInputs, + partition_proofs: &[Self::Proof], + sector_idxs: &[u64], + ) -> Result { + let challenge_count = pub_params.challenge_count; + let num_sectors_per_chunk = pub_params.sector_count; + let num_sectors = pub_inputs.sectors.len(); + + ensure!( + num_sectors <= num_sectors_per_chunk * partition_proofs.len(), + "inconsistent number of sectors: {} > {} * {}", + num_sectors, + num_sectors_per_chunk, + partition_proofs.len(), + ); + + for (j, (proof, pub_sectors_chunk)) in partition_proofs + .iter() + .zip(pub_inputs.sectors.chunks(num_sectors_per_chunk)) + .enumerate() + { + ensure!( + pub_sectors_chunk.len() <= num_sectors_per_chunk, + "inconsistent number of public sectors: {} > {}", + pub_sectors_chunk.len(), + num_sectors_per_chunk, + ); + ensure!( + proof.sectors.len() == num_sectors_per_chunk, + "invalid number of sectors in the partition proof {}: {} != {}", + j, + proof.sectors.len(), + num_sectors_per_chunk, + ); + + let is_valid = pub_sectors_chunk + .par_iter() + .zip(proof.sectors.par_iter()) + .zip(sector_idxs.par_iter()) + .enumerate() + .map(|(_i, ((pub_sector, sector_proof), sector_idx_u64))| { + let sector_id = pub_sector.id; + let comm_r = &pub_sector.comm_r; + let comm_c = sector_proof.comm_c; + let inclusion_proofs = §or_proof.inclusion_proofs; + + // Verify that H(Comm_c || Comm_r_last) == Comm_R + + // comm_r_last is the root of the proof + let comm_r_last = inclusion_proofs[0].root(); + + if AsRef::<[u8]>::as_ref(&::Function::hash2( + &comm_c, + &comm_r_last, + )) != AsRef::<[u8]>::as_ref(comm_r) + { + error!("hash(comm_c || comm_r_last) != comm_r: {:?}", sector_id); + return Ok(false); + } + + ensure!( + challenge_count == inclusion_proofs.len(), + "unexpected number of inclusion proofs: {} != {}", + challenge_count, + inclusion_proofs.len() + ); + + // avoid rehashing fixed inputs + let mut challenge_hasher = Sha256::new(); + challenge_hasher.update(AsRef::<[u8]>::as_ref(&pub_inputs.randomness)); + challenge_hasher.update(&u64::from(sector_id).to_le_bytes()[..]); + + let sector_idx = *sector_idx_u64 as usize; + let partition_idx_in_window = sector_idx / num_sectors_per_chunk; + let sector_idx_in_partition = sector_idx % num_sectors_per_chunk; + let is_valid_list = inclusion_proofs + .par_iter() + .enumerate() + .map(|(n, inclusion_proof)| -> Result { + let challenge_index = (partition_idx_in_window * num_sectors_per_chunk + + sector_idx_in_partition) + * pub_params.challenge_count + + n; + let challenged_leaf = + generate_leaf_challenge_inner::<::Domain>( + challenge_hasher.clone(), + pub_params, + challenge_index as u64, + ); + + // validate all comm_r_lasts match + if inclusion_proof.root() != comm_r_last { + error!("inclusion proof root != comm_r_last: {:?}", sector_id); + return Ok(false); + } + + // validate the path length + let expected_path_length = inclusion_proof + .expected_len(pub_params.sector_size as usize / NODE_SIZE); + + if expected_path_length != inclusion_proof.path().len() { + error!("wrong path length: {:?}", sector_id); + return Ok(false); + } + + if !inclusion_proof.validate(challenged_leaf as usize) { + error!("invalid inclusion proof: {:?}", sector_id); + return Ok(false); + } + Ok(true) + }) + .collect::>>()?; + + Ok(is_valid_list.into_iter().all(|v| v)) + }) + .reduce( + || Ok(true), + |all_valid, is_valid| Ok(all_valid? && is_valid?), + )?; + if !is_valid { + return Ok(false); + } + } + Ok(true) + } }