From 01ce54ea107051e2d5398f6425953472d988b835 Mon Sep 17 00:00:00 2001 From: gegezai Date: Fri, 27 Aug 2021 14:33:14 +0800 Subject: [PATCH] * init decoupling-post-worker code --- filecoin-proofs/src/api/post_util.rs | 139 +++++++++++++++++++- filecoin-proofs/src/api/window_post.rs | 75 ++++++++++- storage-proofs-core/src/compound_proof.rs | 7 + storage-proofs-core/src/proof.rs | 17 +++ storage-proofs-post/src/fallback/vanilla.rs | 130 ++++++++++++++++++ 5 files changed, 366 insertions(+), 2 deletions(-) diff --git a/filecoin-proofs/src/api/post_util.rs b/filecoin-proofs/src/api/post_util.rs index 1c76fd96e5..27fe24bb5e 100644 --- a/filecoin-proofs/src/api/post_util.rs +++ b/filecoin-proofs/src/api/post_util.rs @@ -7,7 +7,8 @@ use bincode::deserialize; use filecoin_hashers::Hasher; use log::{info, trace}; use storage_proofs_core::{ - cache_key::CacheKey, merkle::MerkleTreeTrait, proof::ProofScheme, sector::SectorId, + cache_key::CacheKey, compound_proof::PubVanRsp, merkle::MerkleTreeTrait, proof::ProofScheme, + sector::SectorId, }; use storage_proofs_post::fallback::{self, generate_leaf_challenge, FallbackPoSt, SectorProof}; @@ -308,3 +309,139 @@ pub(crate) fn get_partitions_for_window_post( None } } + +pub fn get_post_vanilla_params( + post_config: &PoStConfig, +) -> Result { + let rsp = PubVanRsp { + challengecount: post_config.challenge_count, + sectorcount: post_config.sector_count, + }; + + Ok(rsp) +} + +pub fn single_partition_vanilla_proofs( + post_config: &PoStConfig, + pub_params: &fallback::PublicParams, + pub_inputs: &fallback::PublicInputs<::Domain>, + partition_count: usize, + vanilla_proofs: &[FallbackPoStSectorProof], + sector_idxs: &[u64], +) -> Result>> { + info!("single_partition_vanilla_proofs:start"); + ensure!( + post_config.typ == PoStType::Window || post_config.typ == PoStType::Winning, + "invalid post config type" + ); + + let num_sectors_per_chunk = pub_params.sector_count; + let num_sectors = pub_inputs.sectors.len(); + + ensure!( + num_sectors <= partition_count * num_sectors_per_chunk, + "cannot prove the provided number of sectors: {} > {} * {}", + num_sectors, + partition_count, + num_sectors_per_chunk, + ); + + let mut partition_proofs = Vec::new(); + + // Note that the partition proofs returned are shaped differently + // based on which type of PoSt is being considered. + match post_config.typ { + PoStType::Window => { + for (j, sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + let mut sector_proofs = Vec::with_capacity(num_sectors_per_chunk); + + for pub_sector in sectors_chunk.iter() { + let cur_proof = vanilla_proofs + .iter() + .find(|&proof| proof.sector_id == pub_sector.id) + .expect("failed to locate sector proof"); + + // Note: Window post requires all inclusion proofs (based on the challenge + // count per sector) per sector proof. + sector_proofs.extend(cur_proof.vanilla_proof.sectors.clone()); + } + + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while sector_proofs.len() < num_sectors_per_chunk { + sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone()); + } + + partition_proofs.push(fallback::Proof::<::Proof> { + sectors: sector_proofs, + }); + } + } + PoStType::Winning => { + for (j, sectors_chunk) in vanilla_proofs.chunks(num_sectors_per_chunk).enumerate() { + trace!("processing partition {}", j); + + // Sanity check incoming structure + ensure!( + sectors_chunk.len() == 1, + "Invalid sector chunk for Winning PoSt" + ); + ensure!( + sectors_chunk[0].vanilla_proof.sectors.len() == 1, + "Invalid sector count for Winning PoSt chunk" + ); + + // Winning post sector_count is winning post challenges per sector + ensure!( + post_config.sector_count == sectors_chunk[j].vanilla_proof.sectors.len(), + "invalid number of sector proofs for Winning PoSt" + ); + + let mut sector_proofs = Vec::with_capacity(post_config.challenge_count); + let cur_sector_proof = §ors_chunk[0].vanilla_proof.sectors[0]; + + // Unroll inclusions proofs from the single provided sector_proof (per partition) + // into individual sector proofs, required for winning post. + for cur_inclusion_proof in cur_sector_proof.inclusion_proofs() { + sector_proofs.push(SectorProof { + inclusion_proofs: vec![cur_inclusion_proof.clone()], + comm_c: cur_sector_proof.comm_c, + comm_r_last: cur_sector_proof.comm_r_last, + }); + } + + // If there were less than the required number of sectors provided, we duplicate the last one + // to pad the proof out, such that it works in the circuit part. + while sector_proofs.len() < num_sectors_per_chunk { + sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone()); + } + + // Winning post Challenge count is the total winning post challenges + ensure!( + sector_proofs.len() == post_config.challenge_count, + "invalid number of partition proofs based on Winning PoSt challenges" + ); + + partition_proofs.push(fallback::Proof::<::Proof> { + sectors: sector_proofs, + }); + } + } + } + + info!("single_partition_vanilla_proofs:finish"); + + ensure!( + FallbackPoSt::::verify_single_partitions( + pub_params, + pub_inputs, + &partition_proofs, + sector_idxs + )?, + "partitioned vanilla proofs failed to verify" + ); + + Ok(partition_proofs) +} diff --git a/filecoin-proofs/src/api/window_post.rs b/filecoin-proofs/src/api/window_post.rs index 99c1356ed5..ab4ba49671 100644 --- a/filecoin-proofs/src/api/window_post.rs +++ b/filecoin-proofs/src/api/window_post.rs @@ -14,7 +14,10 @@ use storage_proofs_post::fallback::{ }; use crate::{ - api::{as_safe_commitment, get_partitions_for_window_post, partition_vanilla_proofs}, + api::{ + as_safe_commitment, get_partitions_for_window_post, partition_vanilla_proofs, + single_partition_vanilla_proofs, + }, caches::{get_post_params, get_post_verifying_key}, parameters::window_post_setup_params, types::{ @@ -242,3 +245,73 @@ pub fn verify_window_post( Ok(true) } + +/// Generates a Window proof-of-spacetime with provided vanilla proofs of a single partition +pub fn generate_single_window_post_with_vanilla( + post_config: &PoStConfig, + randomness: &ChallengeSeed, + prover_id: ProverId, + vanilla_proofs: Vec>, + sector_idxs: &[u64], +) -> Result { + info!("generate_single_window_post_with_vanilla:start"); + ensure!( + post_config.typ == PoStType::Window, + "invalid post config type" + ); + + let randomness_safe: ::Domain = + as_safe_commitment(randomness, "randomness")?; + let prover_id_safe: ::Domain = + as_safe_commitment(&prover_id, "prover_id")?; + + let vanilla_params = window_post_setup_params(&post_config); + let partitions = get_partitions_for_window_post(vanilla_proofs.len(), &post_config); + + let setup_params = compound_proof::SetupParams { + vanilla_params, + partitions, + priority: post_config.priority, + }; + + let partitions = partitions.unwrap_or(1); + + let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, Tree>> = + FallbackPoStCompound::setup(&setup_params)?; + let groth_params = get_post_params::(&post_config)?; + + let mut pub_sectors = Vec::with_capacity(vanilla_proofs.len()); + for vanilla_proof in &vanilla_proofs { + pub_sectors.push(PublicSector { + id: vanilla_proof.sector_id, + comm_r: vanilla_proof.comm_r, + }); + } + + let pub_inputs = fallback::PublicInputs { + randomness: randomness_safe, + prover_id: prover_id_safe, + sectors: pub_sectors, + k: None, + }; + + let partitioned_proofs = single_partition_vanilla_proofs( + &post_config, + &pub_params.vanilla_params, + &pub_inputs, + partitions, + &vanilla_proofs, + sector_idxs, + )?; + + let proof = FallbackPoStCompound::prove_with_vanilla( + &pub_params, + &pub_inputs, + partitioned_proofs, + &groth_params, + )?; + + info!("generate_single_window_post_with_vanilla:finish"); + + proof.to_vec() +} diff --git a/storage-proofs-core/src/compound_proof.rs b/storage-proofs-core/src/compound_proof.rs index 6532a835fb..4823a74360 100644 --- a/storage-proofs-core/src/compound_proof.rs +++ b/storage-proofs-core/src/compound_proof.rs @@ -16,6 +16,7 @@ use rand::{rngs::OsRng, RngCore}; use rayon::prelude::{ IndexedParallelIterator, IntoParallelIterator, IntoParallelRefIterator, ParallelIterator, }; +use serde::{Deserialize, Serialize}; use crate::{ error::Result, @@ -497,3 +498,9 @@ where Ok(res) } } + +#[derive(Serialize, Deserialize)] +pub struct PubVanRsp { + pub challengecount: usize, + pub sectorcount: usize, +} diff --git a/storage-proofs-core/src/proof.rs b/storage-proofs-core/src/proof.rs index 2b39a9fc69..9f6320cde9 100644 --- a/storage-proofs-core/src/proof.rs +++ b/storage-proofs-core/src/proof.rs @@ -85,6 +85,23 @@ pub trait ProofScheme<'a> { Ok(true) } + fn verify_single_partitions( + pub_params: &Self::PublicParams, + pub_in: &Self::PublicInputs, + proofs: &[Self::Proof], + _sector_idxs: &[u64], + ) -> Result { + for (k, proof) in proofs.iter().enumerate() { + let partition_pub_in = Self::with_partition((*pub_in).clone(), Some(k)); // + + if !Self::verify(pub_params, &partition_pub_in, proof)? { + return Ok(false); + } + } + + Ok(true) + } + // This method must be specialized by concrete ProofScheme implementations which use partitions. fn with_partition(pub_in: Self::PublicInputs, _k: Option) -> Self::PublicInputs { pub_in diff --git a/storage-proofs-post/src/fallback/vanilla.rs b/storage-proofs-post/src/fallback/vanilla.rs index ac7441b7a6..cbce1d8312 100644 --- a/storage-proofs-post/src/fallback/vanilla.rs +++ b/storage-proofs-post/src/fallback/vanilla.rs @@ -630,4 +630,134 @@ impl<'a, Tree: 'a + MerkleTreeTrait> ProofScheme<'a> for FallbackPoSt<'a, Tree> checked * public_params.challenge_count >= requirements.minimum_challenge_count } + + // Verify vanilla proofs for a single partition. + // Calculate the offset for the challenge position of the sector + fn verify_single_partitions( + pub_params: &Self::PublicParams, + pub_inputs: &Self::PublicInputs, + partition_proofs: &[Self::Proof], + sector_idxs: &[u64], + ) -> Result { + let challenge_count = pub_params.challenge_count; + let num_sectors_per_chunk = pub_params.sector_count; + let num_sectors = pub_inputs.sectors.len(); + + ensure!( + num_sectors <= num_sectors_per_chunk * partition_proofs.len(), + "inconsistent number of sectors: {} > {} * {}", + num_sectors, + num_sectors_per_chunk, + partition_proofs.len(), + ); + + for (j, (proof, pub_sectors_chunk)) in partition_proofs + .iter() + .zip(pub_inputs.sectors.chunks(num_sectors_per_chunk)) + .enumerate() + { + ensure!( + pub_sectors_chunk.len() <= num_sectors_per_chunk, + "inconsistent number of public sectors: {} > {}", + pub_sectors_chunk.len(), + num_sectors_per_chunk, + ); + ensure!( + proof.sectors.len() == num_sectors_per_chunk, + "invalid number of sectors in the partition proof {}: {} != {}", + j, + proof.sectors.len(), + num_sectors_per_chunk, + ); + + let is_valid = pub_sectors_chunk + .par_iter() + .zip(proof.sectors.par_iter()) + .zip(sector_idxs.par_iter()) + .enumerate() + .map(|(_i, ((pub_sector, sector_proof), sector_idx_u64))| { + let sector_id = pub_sector.id; + let comm_r = &pub_sector.comm_r; + let comm_c = sector_proof.comm_c; + let inclusion_proofs = §or_proof.inclusion_proofs; + + // Verify that H(Comm_c || Comm_r_last) == Comm_R + + // comm_r_last is the root of the proof + let comm_r_last = inclusion_proofs[0].root(); + + if AsRef::<[u8]>::as_ref(&::Function::hash2( + &comm_c, + &comm_r_last, + )) != AsRef::<[u8]>::as_ref(comm_r) + { + error!("hash(comm_c || comm_r_last) != comm_r: {:?}", sector_id); + return Ok(false); + } + + ensure!( + challenge_count == inclusion_proofs.len(), + "unexpected number of inclusion proofs: {} != {}", + challenge_count, + inclusion_proofs.len() + ); + + // avoid rehashing fixed inputs + let mut challenge_hasher = Sha256::new(); + challenge_hasher.update(AsRef::<[u8]>::as_ref(&pub_inputs.randomness)); + challenge_hasher.update(&u64::from(sector_id).to_le_bytes()[..]); + + let sector_idx = *sector_idx_u64 as usize; + let partition_idx_in_window = sector_idx / num_sectors_per_chunk; + let sector_idx_in_partition = sector_idx % num_sectors_per_chunk; + let is_valid_list = inclusion_proofs + .par_iter() + .enumerate() + .map(|(n, inclusion_proof)| -> Result { + let challenge_index = (partition_idx_in_window * num_sectors_per_chunk + + sector_idx_in_partition) + * pub_params.challenge_count + + n; + let challenged_leaf = + generate_leaf_challenge_inner::<::Domain>( + challenge_hasher.clone(), + pub_params, + challenge_index as u64, + ); + + // validate all comm_r_lasts match + if inclusion_proof.root() != comm_r_last { + error!("inclusion proof root != comm_r_last: {:?}", sector_id); + return Ok(false); + } + + // validate the path length + let expected_path_length = inclusion_proof + .expected_len(pub_params.sector_size as usize / NODE_SIZE); + + if expected_path_length != inclusion_proof.path().len() { + error!("wrong path length: {:?}", sector_id); + return Ok(false); + } + + if !inclusion_proof.validate(challenged_leaf as usize) { + error!("invalid inclusion proof: {:?}", sector_id); + return Ok(false); + } + Ok(true) + }) + .collect::>>()?; + + Ok(is_valid_list.into_iter().all(|v| v)) + }) + .reduce( + || Ok(true), + |all_valid, is_valid| Ok(all_valid? && is_valid?), + )?; + if !is_valid { + return Ok(false); + } + } + Ok(true) + } }