feat: allow for window post proving on a partition basis (#1526)

* * init decoupling-post-worker code
* unify compound verification
* reduce code duplication in partitioning
* simplify partition count retrieval
* feat: attempt to generate parameters on OS X
* feat: make PartitionSnarkProof clone-able
* fix: apply review feedback

Co-authored-by: gegezai <gegezai@mingzhutek.com>
Co-authored-by: dignifiedquire <me@dignifiedquire.com>
Co-authored-by: nemo <nemo@protocol.ai>

.circleci/config.yml

@@ -294,6 +294,7 @@ jobs:
       - run: rustup default $(cat rust-toolchain)
       - run: cargo update
       - run: cargo fetch
+      - ensure_filecoin_parameters
       - run:
           name: Test Darwin
           command: |

filecoin-proofs/src/api/post_util.rs

@@ -5,7 +5,7 @@ use std::path::Path;
 use anyhow::{anyhow, ensure, Context, Result};
 use bincode::deserialize;
 use filecoin_hashers::Hasher;
-use log::{info, trace};
+use log::{debug, info};
 use storage_proofs_core::{
     cache_key::CacheKey, merkle::MerkleTreeTrait, proof::ProofScheme, sector::SectorId,
 };
@@ -18,7 +18,7 @@ use crate::{
         ChallengeSeed, FallbackPoStSectorProof, PoStConfig, PrivateReplicaInfo, ProverId,
         TemporaryAux, VanillaProof,
     },
-    PoStType,
+    PartitionSnarkProof, PoStType, SnarkProof, SINGLE_PARTITION_PROOF_LEN,
 };
 
 // Ensure that any associated cached data persisted is discarded.
@@ -208,80 +208,29 @@ pub fn partition_vanilla_proofs<Tree: MerkleTreeTrait>(
     match post_config.typ {
         PoStType::Window => {
             for (j, sectors_chunk) in pub_inputs.sectors.chunks(num_sectors_per_chunk).enumerate() {
-                trace!("processing partition {}", j);
-
-                let mut sector_proofs = Vec::with_capacity(num_sectors_per_chunk);
-
-                for pub_sector in sectors_chunk.iter() {
-                    let cur_proof = vanilla_proofs
-                        .iter()
-                        .find(|&proof| proof.sector_id == pub_sector.id)
-                        .expect("failed to locate sector proof");
-
-                    // Note: Window post requires all inclusion proofs (based on the challenge
-                    // count per sector) per sector proof.
-                    sector_proofs.extend(cur_proof.vanilla_proof.sectors.clone());
-                }
-
-                // If there were less than the required number of sectors provided, we duplicate the last one
-                // to pad the proof out, such that it works in the circuit part.
-                while sector_proofs.len() < num_sectors_per_chunk {
-                    sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone());
-                }
-
-                partition_proofs.push(fallback::Proof::<<Tree as MerkleTreeTrait>::Proof> {
-                    sectors: sector_proofs,
-                });
+                let proof = single_partition_vanilla_proofs(
+                    post_config,
+                    pub_params,
+                    &fallback::PublicInputs {
+                        randomness: pub_inputs.randomness,
+                        prover_id: pub_inputs.prover_id,
+                        sectors: sectors_chunk.to_vec(),
+                        k: Some(j),
+                    },
+                    vanilla_proofs,
+                )?;
+                partition_proofs.push(proof);
             }
         }
         PoStType::Winning => {
             for (j, sectors_chunk) in vanilla_proofs.chunks(num_sectors_per_chunk).enumerate() {
-                trace!("processing partition {}", j);
-
-                // Sanity check incoming structure
-                ensure!(
-                    sectors_chunk.len() == 1,
-                    "Invalid sector chunk for Winning PoSt"
-                );
-                ensure!(
-                    sectors_chunk[0].vanilla_proof.sectors.len() == 1,
-                    "Invalid sector count for Winning PoSt chunk"
-                );
-
-                // Winning post sector_count is winning post challenges per sector
-                ensure!(
-                    post_config.sector_count == sectors_chunk[j].vanilla_proof.sectors.len(),
-                    "invalid number of sector proofs for Winning PoSt"
-                );
-
-                let mut sector_proofs = Vec::with_capacity(post_config.challenge_count);
-                let cur_sector_proof = &sectors_chunk[0].vanilla_proof.sectors[0];
-
-                // Unroll inclusions proofs from the single provided sector_proof (per partition)
-                // into individual sector proofs, required for winning post.
-                for cur_inclusion_proof in cur_sector_proof.inclusion_proofs() {
-                    sector_proofs.push(SectorProof {
-                        inclusion_proofs: vec![cur_inclusion_proof.clone()],
-                        comm_c: cur_sector_proof.comm_c,
-                        comm_r_last: cur_sector_proof.comm_r_last,
-                    });
-                }
-
-                // If there were less than the required number of sectors provided, we duplicate the last one
-                // to pad the proof out, such that it works in the circuit part.
-                while sector_proofs.len() < num_sectors_per_chunk {
-                    sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone());
-                }
-
-                // Winning post Challenge count is the total winning post challenges
-                ensure!(
-                    sector_proofs.len() == post_config.challenge_count,
-                    "invalid number of partition proofs based on Winning PoSt challenges"
-                );
-
-                partition_proofs.push(fallback::Proof::<<Tree as MerkleTreeTrait>::Proof> {
-                    sectors: sector_proofs,
-                });
+                let proof = single_partition_vanilla_proofs(
+                    post_config,
+                    pub_params,
+                    &fallback::FallbackPoSt::<Tree>::with_partition(pub_inputs.clone(), Some(j)),
+                    sectors_chunk,
+                )?;
+                partition_proofs.push(proof);
             }
         }
     }
@@ -308,3 +257,140 @@ pub(crate) fn get_partitions_for_window_post(
         None
     }
 }
+
+pub fn single_partition_vanilla_proofs<Tree: MerkleTreeTrait>(
+    post_config: &PoStConfig,
+    pub_params: &fallback::PublicParams,
+    pub_inputs: &fallback::PublicInputs<<Tree::Hasher as Hasher>::Domain>,
+    vanilla_proofs: &[FallbackPoStSectorProof<Tree>],
+) -> Result<VanillaProof<Tree>> {
+    info!("single_partition_vanilla_proofs:start");
+    ensure!(pub_inputs.k.is_some(), "must have a partition index");
+    let partition_index = pub_inputs.k.expect("prechecked");
+
+    debug!("processing partition: {}", partition_index);
+    ensure!(
+        post_config.typ == PoStType::Window || post_config.typ == PoStType::Winning,
+        "invalid post config type"
+    );
+
+    let num_sectors_per_chunk = pub_params.sector_count;
+    let num_sectors = pub_inputs.sectors.len();
+    ensure!(
+        num_sectors <= num_sectors_per_chunk,
+        "can only prove a single partition"
+    );
+
+    // Note that the partition proofs returned are shaped differently
+    // based on which type of PoSt is being considered.
+    let partition_proof = match post_config.typ {
+        PoStType::Window => {
+            let sectors_chunk = &pub_inputs.sectors;
+
+            let mut sector_proofs = Vec::with_capacity(num_sectors_per_chunk);
+
+            for pub_sector in sectors_chunk.iter() {
+                let cur_proof = vanilla_proofs
+                    .iter()
+                    .find(|&proof| proof.sector_id == pub_sector.id)
+                    .expect("failed to locate sector proof");
+
+                // Note: Window post requires all inclusion proofs (based on the challenge
+                // count per sector) per sector proof.
+                sector_proofs.extend(cur_proof.vanilla_proof.sectors.clone());
+            }
+
+            // If there were less than the required number of sectors provided, we duplicate the last one
+            // to pad the proof out, such that it works in the circuit part.
+            while sector_proofs.len() < num_sectors_per_chunk {
+                sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone());
+            }
+
+            fallback::Proof::<<Tree as MerkleTreeTrait>::Proof> {
+                sectors: sector_proofs,
+            }
+        }
+        PoStType::Winning => {
+            let sectors_chunk = vanilla_proofs;
+            // Sanity check incoming structure
+            ensure!(
+                sectors_chunk.len() == 1,
+                "Invalid sector chunk for Winning PoSt"
+            );
+            ensure!(
+                sectors_chunk[0].vanilla_proof.sectors.len() == 1,
+                "Invalid sector count for Winning PoSt chunk"
+            );
+
+            // Winning post sector_count is winning post challenges per sector
+            ensure!(
+                post_config.sector_count
+                    == sectors_chunk[partition_index].vanilla_proof.sectors.len(),
+                "invalid number of sector proofs for Winning PoSt"
+            );
+
+            let mut sector_proofs = Vec::with_capacity(post_config.challenge_count);
+            let cur_sector_proof = &sectors_chunk[0].vanilla_proof.sectors[0];
+
+            // Unroll inclusions proofs from the single provided sector_proof (per partition)
+            // into individual sector proofs, required for winning post.
+            for cur_inclusion_proof in cur_sector_proof.inclusion_proofs() {
+                sector_proofs.push(SectorProof {
+                    inclusion_proofs: vec![cur_inclusion_proof.clone()],
+                    comm_c: cur_sector_proof.comm_c,
+                    comm_r_last: cur_sector_proof.comm_r_last,
+                });
+            }
+
+            // If there were less than the required number of sectors provided, we duplicate the last one
+            // to pad the proof out, such that it works in the circuit part.
+            while sector_proofs.len() < num_sectors_per_chunk {
+                sector_proofs.push(sector_proofs[sector_proofs.len() - 1].clone());
+            }
+
+            // Winning post Challenge count is the total winning post challenges
+            ensure!(
+                sector_proofs.len() == post_config.challenge_count,
+                "invalid number of partition proofs based on Winning PoSt challenges"
+            );
+
+            fallback::Proof::<<Tree as MerkleTreeTrait>::Proof> {
+                sectors: sector_proofs,
+            }
+        }
+    };
+
+    info!("single_partition_vanilla_proofs:finish");
+
+    ensure!(
+        FallbackPoSt::<Tree>::verify(pub_params, pub_inputs, &partition_proof)?,
+        "partitioned vanilla proofs failed to verify"
+    );
+
+    Ok(partition_proof)
+}
+
+pub fn merge_window_post_partition_proofs(
+    mut proofs: Vec<PartitionSnarkProof>,
+) -> Result<SnarkProof> {
+    let mut proof = Vec::with_capacity(proofs.len() * SINGLE_PARTITION_PROOF_LEN);
+    for p in proofs.iter_mut() {
+        proof.append(&mut p.0);
+    }
+
+    Ok(proof)
+}
+
+pub fn get_num_partition_for_fallback_post(config: &PoStConfig, num_sectors: usize) -> usize {
+    match config.typ {
+        PoStType::Window => {
+            let partitions = (num_sectors as f32 / config.sector_count as f32).ceil() as usize;
+            if partitions > 1 {
+                partitions
+            } else {
+                1
+            }
+        }
+        PoStType::Winning => 1,
+    }
+}

filecoin-proofs/src/api/window_post.rs

@@ -14,14 +14,17 @@ use storage_proofs_post::fallback::{
 };
 
 use crate::{
-    api::{as_safe_commitment, get_partitions_for_window_post, partition_vanilla_proofs},
+    api::{
+        as_safe_commitment, get_partitions_for_window_post, partition_vanilla_proofs,
+        single_partition_vanilla_proofs,
+    },
     caches::{get_post_params, get_post_verifying_key},
     parameters::window_post_setup_params,
     types::{
         ChallengeSeed, FallbackPoStSectorProof, PoStConfig, PrivateReplicaInfo, ProverId,
         PublicReplicaInfo, SnarkProof,
     },
-    PoStType,
+    PartitionSnarkProof, PoStType,
 };
 
 /// Generates a Window proof-of-spacetime with provided vanilla proofs.
@@ -242,3 +245,69 @@ pub fn verify_window_post<Tree: 'static + MerkleTreeTrait>(
 
     Ok(true)
 }
+
+/// Generates a Window proof-of-spacetime with provided vanilla proofs of a single partition.
+pub fn generate_single_window_post_with_vanilla<Tree: 'static + MerkleTreeTrait>(
+    post_config: &PoStConfig,
+    randomness: &ChallengeSeed,
+    prover_id: ProverId,
+    vanilla_proofs: Vec<FallbackPoStSectorProof<Tree>>,
+    partition_index: usize,
+) -> Result<PartitionSnarkProof> {
+    info!("generate_single_window_post_with_vanilla:start");
+    ensure!(
+        post_config.typ == PoStType::Window,
+        "invalid post config type"
+    );
+
+    let randomness_safe: <Tree::Hasher as Hasher>::Domain =
+        as_safe_commitment(randomness, "randomness")?;
+    let prover_id_safe: <Tree::Hasher as Hasher>::Domain =
+        as_safe_commitment(&prover_id, "prover_id")?;
+
+    let vanilla_params = window_post_setup_params(post_config);
+    let partitions = get_partitions_for_window_post(vanilla_proofs.len(), post_config);
+
+    let setup_params = compound_proof::SetupParams {
+        vanilla_params,
+        partitions,
+        priority: post_config.priority,
+    };
+
+    let pub_params: compound_proof::PublicParams<'_, FallbackPoSt<'_, Tree>> =
+        FallbackPoStCompound::setup(&setup_params)?;
+    let groth_params = get_post_params::<Tree>(post_config)?;
+
+    let mut pub_sectors = Vec::with_capacity(vanilla_proofs.len());
+    for vanilla_proof in &vanilla_proofs {
+        pub_sectors.push(PublicSector {
+            id: vanilla_proof.sector_id,
+            comm_r: vanilla_proof.comm_r,
+        });
+    }
+
+    let pub_inputs = fallback::PublicInputs {
+        randomness: randomness_safe,
+        prover_id: prover_id_safe,
+        sectors: pub_sectors,
+        k: Some(partition_index),
+    };
+
+    let partitioned_proofs = single_partition_vanilla_proofs(
+        post_config,
+        &pub_params.vanilla_params,
+        &pub_inputs,
+        &vanilla_proofs,
+    )?;
+
+    let proof = FallbackPoStCompound::prove_with_vanilla(
+        &pub_params,
+        &pub_inputs,
+        vec![partitioned_proofs],
+        &groth_params,
+    )?;
+
+    info!("generate_single_window_post_with_vanilla:finish");
+
+    proof.to_vec().map(PartitionSnarkProof)
+}

filecoin-proofs/src/types/mod.rs

@@ -82,6 +82,10 @@ pub struct SealPreCommitPhase1Output<Tree: MerkleTreeTrait> {
     pub comm_d: Commitment,
 }
 
+#[repr(transparent)]
+#[derive(Clone, Debug)]
+pub struct PartitionSnarkProof(pub Vec<u8>);
+
 pub type SnarkProof = Vec<u8>;
 pub type AggregateSnarkProof = Vec<u8>;
 pub type VanillaProof<Tree> = fallback::Proof<<Tree as MerkleTreeTrait>::Proof>;