From c9e97ee46c51c2d7330ba8cd0f6fae2527d8f648 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Wed, 20 Sep 2023 11:09:38 +0200 Subject: [PATCH 1/4] refactor: clearer Synthetic PoRep separation The Synthetic PoRep code is interleaved with the original PoRep code. This commit refactors the code, so that it's clearer what the differences between Synthetic and Non-Synthetic PoRep is, without the need to read through large parts of the code, looking for if-statements. It also becomes more apparent in the code, that there are two "modes" of the `prove_layers` function in case of the Synthetic PoRep. One generation step in case there's no seed and one extracting step when there's a seed. There no longer is a fallback in case from Synthetic PoRep to Non-Synthetic PoRep in case the Synthetic PoRep Proofs file cannot be read, but when the labels are set. I consider that a feature as if this case happens, your system likelt has some problems, so it's better to error early. --- .../src/stacked/vanilla/proof.rs | 142 +++++++++--------- 1 file changed, 74 insertions(+), 68 deletions(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index 950274f2b..1f0a1fbf2 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -105,66 +105,87 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr partition_count: usize, ) -> Result>>> { assert!(layers > 0); + // Sanity checks on restored trees. + assert!(pub_inputs.tau.is_some()); - if !layer_challenges.use_synthetic { - // This needs to be relaxed now since the layers may not exist in the synth porep case - assert_eq!(t_aux.labels.len(), layers); - } + if layer_challenges.use_synthetic { + // If there are no synthetic vanilla proofs stored on disk yet, generate them. + if pub_inputs.seed.is_none() { + info!("generating synthetic vanilla proofs in a single partition"); - let graph_size = graph.size(); + let vanilla_proofs = Self::prove_layers_generate( + graph, + pub_inputs, + p_aux.comm_c, + t_aux, + layer_challenges, + layers, + partition_count, + )?; - // Sanity checks on restored trees. - assert!(pub_inputs.tau.is_some()); - // Skip this check in the case of synthetic porep - if t_aux.tree_d.is_some() { + assert!( + vanilla_proofs.iter().skip(1).all(Vec::is_empty), + "synthetic proofs should be generated in a single partition", + ); + let synth_proofs = &vanilla_proofs[0]; + Self::write_synth_proofs( + synth_proofs, + pub_inputs, + graph, + layer_challenges, + t_aux.synth_proofs_path(), + )?; + Ok(vec![vec![]; partition_count]) + } + // Else the synthetic vanilla proofs are stored on disk, read and return the proofs + // corresponding to the porep challlenge set. + else { + Self::read_porep_proofs_from_synth( + graph.size(), + pub_inputs, + layer_challenges, + t_aux.synth_proofs_path(), + partition_count, + ) + .map_err(|error| { + info!( + "failed to read porep proofs from synthetic proofs file: {:?}", + t_aux.synth_proofs_path(), + ); + error + }) + } + } else { + assert_eq!(t_aux.labels.len(), layers); assert_eq!( pub_inputs.tau.as_ref().expect("as_ref failure").comm_d, t_aux.tree_d.as_ref().expect("failed to get tree_d").root() ); - } - // If synthetic vanilla proofs are stored on disk, read and return the proofs corresponding - // to the porep challlenge set. - let read_synth_proofs = layer_challenges.use_synthetic && pub_inputs.seed.is_some(); - if read_synth_proofs { - let read_res = Self::read_porep_proofs_from_synth( - graph_size, + info!("generating non-synthetic vanilla proofs"); + + Self::prove_layers_generate( + graph, pub_inputs, - layer_challenges, + p_aux.comm_c, t_aux, + layer_challenges, + layers, partition_count, - ); - if read_res.is_ok() { - return read_res; - } - info!( - "failed to read porep proofs from synthetic proofs file: {:?}", - t_aux.synth_proofs_path(), - ); - - // If the synthetic proofs file does not exist and we have the layers available, - // we can generate non-synthetic proofs - if t_aux.labels.len() == layers { - info!("skipping synthetic proving; generating non-synthetic vanilla proofs"); - } else { - error!("synthetic proving failure; synthetic proofs and layers are unavailable"); - - return read_res; - } - } - - // If generating vanilla proofs for the synthetic challenge set, generate those proofs in a - // single partition (otherwise we must ensure tha the synthetic challenge count is divisible - // by the porep partition count). - let gen_synth_proofs = layer_challenges.use_synthetic && pub_inputs.seed.is_none(); - if gen_synth_proofs { - info!("generating synthetic vanilla proofs in a single partition"); + ) } + } - info!( - "read_synth_porep: {}, gen_synth_porep {}", - read_synth_proofs, gen_synth_proofs - ); + #[allow(clippy::too_many_arguments)] + fn prove_layers_generate( + graph: &StackedBucketGraph, + pub_inputs: &PublicInputs<::Domain, ::Domain>, + comm_c: ::Domain, + t_aux: &TemporaryAuxCache, + layer_challenges: &LayerChallenges, + layers: usize, + partition_count: usize, + ) -> Result>>> { let get_drg_parents_columns = |x: usize| -> Result>> { let base_degree = graph.base_graph().degree(); @@ -195,12 +216,12 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr .collect() }; - let vanilla_proofs = (0..partition_count) + (0..partition_count) .map(|k| { trace!("proving partition {}/{}", k + 1, partition_count); // Derive the set of challenges we are proving over. - let challenges = pub_inputs.challenges(layer_challenges, graph_size, Some(k)); + let challenges = pub_inputs.challenges(layer_challenges, graph.size(), Some(k)); THREAD_POOL.scoped(|scope| { // Stacked commitment specifics @@ -230,7 +251,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr assert!(t_aux.tree_c.is_some()); let tree_c = t_aux.tree_c.as_ref().expect("failed to get tree_c"); - assert_eq!(p_aux.comm_c, tree_c.root()); + assert_eq!(comm_c, tree_c.root()); // All labels in C_X trace!(" c_x"); @@ -355,20 +376,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr .collect() }) }) - .collect::>>>>()?; - - // If synthetic vanilla proofs were generated, persist them here. - if gen_synth_proofs { - assert!( - vanilla_proofs.iter().skip(1).all(Vec::is_empty), - "synthetic proofs should be generated in a single partition", - ); - let synth_proofs = &vanilla_proofs[0]; - Self::write_synth_proofs(synth_proofs, pub_inputs, graph, layer_challenges, t_aux)?; - return Ok(vec![vec![]; partition_count]); - } - - Ok(vanilla_proofs) + .collect::>>>>() } fn write_synth_proofs( @@ -376,7 +384,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr pub_inputs: &PublicInputs<::Domain, ::Domain>, graph: &StackedBucketGraph, layer_challenges: &LayerChallenges, - t_aux: &TemporaryAuxCache, + path: PathBuf, ) -> Result<()> { use crate::stacked::vanilla::SynthChallenges; @@ -414,7 +422,6 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr } }); - let path = t_aux.synth_proofs_path(); info!("writing synth-porep vanilla proofs to file: {:?}", path); let file = File::create(&path).map(BufWriter::new).with_context(|| { format!( @@ -439,7 +446,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr sector_nodes: usize, pub_inputs: &PublicInputs<::Domain, ::Domain>, layer_challenges: &LayerChallenges, - t_aux: &TemporaryAuxCache, + path: PathBuf, partition_count: usize, ) -> Result>>> { ensure!( @@ -460,7 +467,6 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr .as_ref() .map(|tau| &tau.comm_r) .expect("unwrapping should not fail"); - let path = t_aux.synth_proofs_path(); info!("reading synthetic vanilla proofs from file: {:?}", path); let num_layers = layer_challenges.layers(); From 1cb3582349c74d3c14d082ea18425674808f32f5 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Mon, 2 Oct 2023 14:48:05 +0200 Subject: [PATCH 2/4] refactor: make prove_layers_generate operate on a single partition `prove_layers_generate()` now operates on a single partition. This makes it clearer that Synthetic PoReps always operate on a single partition only. With the challenge generation moved outside of this function, it's also a clearer separation of concerns, as it now operates directly on the challenges given, without know anything about how they were generated. This also useful for the work of having small, special case binaries for certain operations. --- .../src/stacked/vanilla/proof.rs | 365 +++++++++--------- 1 file changed, 187 insertions(+), 178 deletions(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index 1f0a1fbf2..fed225415 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -112,24 +112,27 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr // If there are no synthetic vanilla proofs stored on disk yet, generate them. if pub_inputs.seed.is_none() { info!("generating synthetic vanilla proofs in a single partition"); + assert_eq!(partition_count, 1); - let vanilla_proofs = Self::prove_layers_generate( + let comm_r = pub_inputs.tau.as_ref().expect("tau is set").comm_r; + // Derive the set of challenges we are proving over. + let challenges = layer_challenges.derive_synthetic( + graph.size(), + &pub_inputs.replica_id, + &comm_r, + ); + + let synth_proofs = Self::prove_layers_generate( graph, pub_inputs, p_aux.comm_c, t_aux, - layer_challenges, + challenges, layers, - partition_count, )?; - assert!( - vanilla_proofs.iter().skip(1).all(Vec::is_empty), - "synthetic proofs should be generated in a single partition", - ); - let synth_proofs = &vanilla_proofs[0]; Self::write_synth_proofs( - synth_proofs, + &synth_proofs, pub_inputs, graph, layer_challenges, @@ -156,36 +159,53 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr }) } } else { - assert_eq!(t_aux.labels.len(), layers); - assert_eq!( - pub_inputs.tau.as_ref().expect("as_ref failure").comm_d, - t_aux.tree_d.as_ref().expect("failed to get tree_d").root() - ); - info!("generating non-synthetic vanilla proofs"); - Self::prove_layers_generate( - graph, - pub_inputs, - p_aux.comm_c, - t_aux, - layer_challenges, - layers, - partition_count, - ) + let comm_r = pub_inputs.tau.as_ref().expect("tau is set").comm_r; + let seed = pub_inputs + .seed + .expect("seed must be set for non-synthetic vanilla proofs"); + + (0..partition_count) + .map(|k| { + trace!("proving partition {}/{}", k + 1, partition_count); + + // Derive the set of challenges we are proving over. + let challenges = layer_challenges.derive( + graph.size(), + &pub_inputs.replica_id, + &comm_r, + &seed, + k as u8, + ); + + Self::prove_layers_generate( + graph, + pub_inputs, + p_aux.comm_c, + t_aux, + challenges, + layers, + ) + }) + .collect::>>>>() } } - #[allow(clippy::too_many_arguments)] fn prove_layers_generate( graph: &StackedBucketGraph, pub_inputs: &PublicInputs<::Domain, ::Domain>, comm_c: ::Domain, t_aux: &TemporaryAuxCache, - layer_challenges: &LayerChallenges, + challenges: Vec, layers: usize, - partition_count: usize, - ) -> Result>>> { + ) -> Result>> { + assert_eq!(t_aux.labels.len(), layers); + assert_eq!( + pub_inputs.tau.as_ref().expect("as_ref failure").comm_d, + t_aux.tree_d.as_ref().expect("failed to get tree_d").root() + ); + let get_drg_parents_columns = |x: usize| -> Result>> { let base_degree = graph.base_graph().degree(); @@ -216,167 +236,156 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr .collect() }; - (0..partition_count) - .map(|k| { - trace!("proving partition {}/{}", k + 1, partition_count); + THREAD_POOL.scoped(|scope| { + // Stacked commitment specifics + challenges + .into_par_iter() + .enumerate() + .map(|(challenge_index, challenge)| { + trace!(" challenge {} ({})", challenge, challenge_index); + assert!(challenge < graph.size(), "Invalid challenge"); + assert!(challenge > 0, "Invalid challenge"); + + let comm_d_proof = t_aux + .tree_d + .as_ref() + .expect("failed to get tree_d") + .gen_proof(challenge)?; + + let comm_d_proof_inner = comm_d_proof.clone(); + let challenge_inner = challenge; + scope.execute(move || { + assert!(comm_d_proof_inner.validate(challenge_inner)); + }); + + // Stacked replica column openings + let rcp = { + let (c_x, drg_parents, exp_parents) = { + assert!(t_aux.tree_c.is_some()); + let tree_c = t_aux.tree_c.as_ref().expect("failed to get tree_c"); + assert_eq!(comm_c, tree_c.root()); + + // All labels in C_X + trace!(" c_x"); + let c_x = t_aux.column(challenge as u32)?.into_proof(tree_c)?; + + // All labels in the DRG parents. + trace!(" drg_parents"); + let drg_parents = get_drg_parents_columns(challenge)? + .into_iter() + .map(|column| column.into_proof(tree_c)) + .collect::>()?; + + // Labels for the expander parents + trace!(" exp_parents"); + let exp_parents = get_exp_parents_columns(challenge)? + .into_iter() + .map(|column| column.into_proof(tree_c)) + .collect::>()?; + + (c_x, drg_parents, exp_parents) + }; - // Derive the set of challenges we are proving over. - let challenges = pub_inputs.challenges(layer_challenges, graph.size(), Some(k)); + ReplicaColumnProof { + c_x, + drg_parents, + exp_parents, + } + }; - THREAD_POOL.scoped(|scope| { - // Stacked commitment specifics - challenges - .into_par_iter() - .enumerate() - .map(|(challenge_index, challenge)| { - trace!(" challenge {} ({})", challenge, challenge_index); - assert!(challenge < graph.size(), "Invalid challenge"); - assert!(challenge > 0, "Invalid challenge"); - - let comm_d_proof = t_aux - .tree_d - .as_ref() - .expect("failed to get tree_d") - .gen_proof(challenge)?; - - let comm_d_proof_inner = comm_d_proof.clone(); - let challenge_inner = challenge; - scope.execute(move || { - assert!(comm_d_proof_inner.validate(challenge_inner)); - }); + // Final replica layer openings + trace!("final replica layer openings"); + let comm_r_last_proof = t_aux.tree_r_last.gen_cached_proof( + challenge, + Some(t_aux.tree_r_last_config_rows_to_discard), + )?; + + let comm_r_last_proof_inner = comm_r_last_proof.clone(); + scope.execute(move || { + debug_assert!(comm_r_last_proof_inner.validate(challenge)); + }); + + // Labeling Proofs Layer 1..l + let mut labeling_proofs = Vec::with_capacity(layers); + let mut encoding_proof = None; + + for layer in 1..=layers { + trace!(" encoding proof layer {}", layer,); + let parents_data: Vec<::Domain> = if layer == 1 { + let mut parents = vec![0; graph.base_graph().degree()]; + graph.base_parents(challenge, &mut parents)?; + + parents + .into_par_iter() + .map(|parent| t_aux.domain_node_at_layer(layer, parent)) + .collect::>()? + } else { + let mut parents = vec![0; graph.degree()]; + graph.parents(challenge, &mut parents)?; + let base_parents_count = graph.base_graph().degree(); + + parents + .into_par_iter() + .enumerate() + .map(|(i, parent)| { + if i < base_parents_count { + // parents data for base parents is from the current layer + t_aux.domain_node_at_layer(layer, parent) + } else { + // parents data for exp parents is from the previous layer + t_aux.domain_node_at_layer(layer - 1, parent) + } + }) + .collect::>()? + }; - // Stacked replica column openings - let rcp = { - let (c_x, drg_parents, exp_parents) = { - assert!(t_aux.tree_c.is_some()); - let tree_c = - t_aux.tree_c.as_ref().expect("failed to get tree_c"); - assert_eq!(comm_c, tree_c.root()); - - // All labels in C_X - trace!(" c_x"); - let c_x = t_aux.column(challenge as u32)?.into_proof(tree_c)?; - - // All labels in the DRG parents. - trace!(" drg_parents"); - let drg_parents = get_drg_parents_columns(challenge)? - .into_iter() - .map(|column| column.into_proof(tree_c)) - .collect::>()?; - - // Labels for the expander parents - trace!(" exp_parents"); - let exp_parents = get_exp_parents_columns(challenge)? - .into_iter() - .map(|column| column.into_proof(tree_c)) - .collect::>()?; - - (c_x, drg_parents, exp_parents) - }; - - ReplicaColumnProof { - c_x, - drg_parents, - exp_parents, - } - }; + // repeat parents + let mut parents_data_full = vec![Default::default(); TOTAL_PARENTS]; + for chunk in parents_data_full.chunks_mut(parents_data.len()) { + chunk.copy_from_slice(&parents_data[..chunk.len()]); + } - // Final replica layer openings - trace!("final replica layer openings"); - let comm_r_last_proof = t_aux.tree_r_last.gen_cached_proof( - challenge, - Some(t_aux.tree_r_last_config_rows_to_discard), - )?; + let proof = LabelingProof::::new( + layer as u32, + challenge as u64, + parents_data_full.clone(), + ); - let comm_r_last_proof_inner = comm_r_last_proof.clone(); + { + let labeled_node = *rcp.c_x.get_node_at_layer(layer)?; + let replica_id = &pub_inputs.replica_id; + let proof_inner = proof.clone(); scope.execute(move || { - debug_assert!(comm_r_last_proof_inner.validate(challenge)); - }); - - // Labeling Proofs Layer 1..l - let mut labeling_proofs = Vec::with_capacity(layers); - let mut encoding_proof = None; - - for layer in 1..=layers { - trace!(" encoding proof layer {}", layer,); - let parents_data: Vec<::Domain> = - if layer == 1 { - let mut parents = vec![0; graph.base_graph().degree()]; - graph.base_parents(challenge, &mut parents)?; - - parents - .into_par_iter() - .map(|parent| t_aux.domain_node_at_layer(layer, parent)) - .collect::>()? - } else { - let mut parents = vec![0; graph.degree()]; - graph.parents(challenge, &mut parents)?; - let base_parents_count = graph.base_graph().degree(); - - parents - .into_par_iter() - .enumerate() - .map(|(i, parent)| { - if i < base_parents_count { - // parents data for base parents is from the current layer - t_aux.domain_node_at_layer(layer, parent) - } else { - // parents data for exp parents is from the previous layer - t_aux.domain_node_at_layer(layer - 1, parent) - } - }) - .collect::>()? - }; - - // repeat parents - let mut parents_data_full = vec![Default::default(); TOTAL_PARENTS]; - for chunk in parents_data_full.chunks_mut(parents_data.len()) { - chunk.copy_from_slice(&parents_data[..chunk.len()]); - } - - let proof = LabelingProof::::new( - layer as u32, - challenge as u64, - parents_data_full.clone(), + assert!( + proof_inner.verify(replica_id, &labeled_node), + "Invalid encoding proof generated at layer {}", + layer, ); + trace!("Valid encoding proof generated at layer {}", layer); + }); + } - { - let labeled_node = *rcp.c_x.get_node_at_layer(layer)?; - let replica_id = &pub_inputs.replica_id; - let proof_inner = proof.clone(); - scope.execute(move || { - assert!( - proof_inner.verify(replica_id, &labeled_node), - "Invalid encoding proof generated at layer {}", - layer, - ); - trace!("Valid encoding proof generated at layer {}", layer); - }); - } - - labeling_proofs.push(proof); + labeling_proofs.push(proof); - if layer == layers { - encoding_proof = Some(EncodingProof::new( - layer as u32, - challenge as u64, - parents_data_full, - )); - } - } + if layer == layers { + encoding_proof = Some(EncodingProof::new( + layer as u32, + challenge as u64, + parents_data_full, + )); + } + } - Ok(Proof { - comm_d_proofs: comm_d_proof, - replica_column_proofs: rcp, - comm_r_last_proof, - labeling_proofs, - encoding_proof: encoding_proof.expect("invalid tapering"), - }) - }) - .collect() + Ok(Proof { + comm_d_proofs: comm_d_proof, + replica_column_proofs: rcp, + comm_r_last_proof, + labeling_proofs, + encoding_proof: encoding_proof.expect("invalid tapering"), + }) }) - }) - .collect::>>>>() + .collect() + }) } fn write_synth_proofs( From 995c0c3676c4c8dd7934b127b0f04221052b048a Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Mon, 9 Oct 2023 13:24:10 +0200 Subject: [PATCH 3/4] docs: clarify challenge generation a bit --- storage-proofs-porep/src/stacked/vanilla/params.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/params.rs b/storage-proofs-porep/src/stacked/vanilla/params.rs index f156d5ec0..e4cd7cc1f 100644 --- a/storage-proofs-porep/src/stacked/vanilla/params.rs +++ b/storage-proofs-porep/src/stacked/vanilla/params.rs @@ -132,9 +132,9 @@ pub struct PublicInputs { impl PublicInputs { /// If the porep challenge randomness `self.seed` is set, this method returns the porep - /// challenges for partition `k`; otherwise if `self.seed` is `None`, returns the entire - /// synthetic challenge set. Note synthetic challenges are generated in a single partition - /// `k = 0`. + /// challenges for partition `k` (for syth and non-syth poreps); otherwise if `self.seed` is + /// `None`, returns the entire synthetic challenge set. Note synthetic challenges are generated + /// in a single partition `k = 0`. pub fn challenges( &self, layer_challenges: &LayerChallenges, From c40fafab7726177851384a54aa6d541b5508a667 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Thu, 23 Nov 2023 11:02:40 +0100 Subject: [PATCH 4/4] chore: fix typo --- storage-proofs-porep/src/stacked/vanilla/params.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/params.rs b/storage-proofs-porep/src/stacked/vanilla/params.rs index e4cd7cc1f..1ecd23168 100644 --- a/storage-proofs-porep/src/stacked/vanilla/params.rs +++ b/storage-proofs-porep/src/stacked/vanilla/params.rs @@ -132,7 +132,7 @@ pub struct PublicInputs { impl PublicInputs { /// If the porep challenge randomness `self.seed` is set, this method returns the porep - /// challenges for partition `k` (for syth and non-syth poreps); otherwise if `self.seed` is + /// challenges for partition `k` (for synth and non-synth poreps); otherwise if `self.seed` is /// `None`, returns the entire synthetic challenge set. Note synthetic challenges are generated /// in a single partition `k = 0`. pub fn challenges(