From 4ffa4a8b6470c027baf3c669bc6a407c8c3999c1 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 1 Dec 2023 14:57:26 +0100 Subject: [PATCH] refactor: clearer Synthetic PoRep separation (#1720) The Synthetic PoRep code is interleaved with the original PoRep code. This commit refactors the code, so that it's clearer what the differences between Synthetic and Non-Synthetic PoRep is, without the need to read through large parts of the code, looking for if-statements. It also becomes more apparent in the code, that there are two "modes" of the `prove_layers` function in case of the Synthetic PoRep. One generation step in case there's no seed and one extracting step when there's a seed. There no longer is a fallback in case from Synthetic PoRep to Non-Synthetic PoRep in case the Synthetic PoRep Proofs file cannot be read, but when the labels are set. I consider that a feature as if this case happens, your system likely has some problems, so it's better to error early. `prove_layers_generate()` now operates on a single partition. This makes it clearer that Synthetic PoReps always operate on a single partition only. With the challenge generation moved outside of this function, it's also a clearer separation of concerns, as it now operates directly on the challenges given, without know anything about how they were generated. This also useful for the work of having small, special case binaries for certain operations. --- .../src/stacked/vanilla/params.rs | 6 +- .../src/stacked/vanilla/proof.rs | 449 +++++++++--------- 2 files changed, 235 insertions(+), 220 deletions(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/params.rs b/storage-proofs-porep/src/stacked/vanilla/params.rs index f156d5ec0..1ecd23168 100644 --- a/storage-proofs-porep/src/stacked/vanilla/params.rs +++ b/storage-proofs-porep/src/stacked/vanilla/params.rs @@ -132,9 +132,9 @@ pub struct PublicInputs { impl PublicInputs { /// If the porep challenge randomness `self.seed` is set, this method returns the porep - /// challenges for partition `k`; otherwise if `self.seed` is `None`, returns the entire - /// synthetic challenge set. Note synthetic challenges are generated in a single partition - /// `k = 0`. + /// challenges for partition `k` (for synth and non-synth poreps); otherwise if `self.seed` is + /// `None`, returns the entire synthetic challenge set. Note synthetic challenges are generated + /// in a single partition `k = 0`. pub fn challenges( &self, layer_challenges: &LayerChallenges, diff --git a/storage-proofs-porep/src/stacked/vanilla/proof.rs b/storage-proofs-porep/src/stacked/vanilla/proof.rs index 950274f2b..fed225415 100644 --- a/storage-proofs-porep/src/stacked/vanilla/proof.rs +++ b/storage-proofs-porep/src/stacked/vanilla/proof.rs @@ -105,66 +105,107 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr partition_count: usize, ) -> Result>>> { assert!(layers > 0); - - if !layer_challenges.use_synthetic { - // This needs to be relaxed now since the layers may not exist in the synth porep case - assert_eq!(t_aux.labels.len(), layers); - } - - let graph_size = graph.size(); - // Sanity checks on restored trees. assert!(pub_inputs.tau.is_some()); - // Skip this check in the case of synthetic porep - if t_aux.tree_d.is_some() { - assert_eq!( - pub_inputs.tau.as_ref().expect("as_ref failure").comm_d, - t_aux.tree_d.as_ref().expect("failed to get tree_d").root() - ); - } - // If synthetic vanilla proofs are stored on disk, read and return the proofs corresponding - // to the porep challlenge set. - let read_synth_proofs = layer_challenges.use_synthetic && pub_inputs.seed.is_some(); - if read_synth_proofs { - let read_res = Self::read_porep_proofs_from_synth( - graph_size, - pub_inputs, - layer_challenges, - t_aux, - partition_count, - ); - if read_res.is_ok() { - return read_res; - } - info!( - "failed to read porep proofs from synthetic proofs file: {:?}", - t_aux.synth_proofs_path(), - ); + if layer_challenges.use_synthetic { + // If there are no synthetic vanilla proofs stored on disk yet, generate them. + if pub_inputs.seed.is_none() { + info!("generating synthetic vanilla proofs in a single partition"); + assert_eq!(partition_count, 1); - // If the synthetic proofs file does not exist and we have the layers available, - // we can generate non-synthetic proofs - if t_aux.labels.len() == layers { - info!("skipping synthetic proving; generating non-synthetic vanilla proofs"); - } else { - error!("synthetic proving failure; synthetic proofs and layers are unavailable"); + let comm_r = pub_inputs.tau.as_ref().expect("tau is set").comm_r; + // Derive the set of challenges we are proving over. + let challenges = layer_challenges.derive_synthetic( + graph.size(), + &pub_inputs.replica_id, + &comm_r, + ); + + let synth_proofs = Self::prove_layers_generate( + graph, + pub_inputs, + p_aux.comm_c, + t_aux, + challenges, + layers, + )?; - return read_res; + Self::write_synth_proofs( + &synth_proofs, + pub_inputs, + graph, + layer_challenges, + t_aux.synth_proofs_path(), + )?; + Ok(vec![vec![]; partition_count]) } - } + // Else the synthetic vanilla proofs are stored on disk, read and return the proofs + // corresponding to the porep challlenge set. + else { + Self::read_porep_proofs_from_synth( + graph.size(), + pub_inputs, + layer_challenges, + t_aux.synth_proofs_path(), + partition_count, + ) + .map_err(|error| { + info!( + "failed to read porep proofs from synthetic proofs file: {:?}", + t_aux.synth_proofs_path(), + ); + error + }) + } + } else { + info!("generating non-synthetic vanilla proofs"); + + let comm_r = pub_inputs.tau.as_ref().expect("tau is set").comm_r; + let seed = pub_inputs + .seed + .expect("seed must be set for non-synthetic vanilla proofs"); + + (0..partition_count) + .map(|k| { + trace!("proving partition {}/{}", k + 1, partition_count); + + // Derive the set of challenges we are proving over. + let challenges = layer_challenges.derive( + graph.size(), + &pub_inputs.replica_id, + &comm_r, + &seed, + k as u8, + ); - // If generating vanilla proofs for the synthetic challenge set, generate those proofs in a - // single partition (otherwise we must ensure tha the synthetic challenge count is divisible - // by the porep partition count). - let gen_synth_proofs = layer_challenges.use_synthetic && pub_inputs.seed.is_none(); - if gen_synth_proofs { - info!("generating synthetic vanilla proofs in a single partition"); + Self::prove_layers_generate( + graph, + pub_inputs, + p_aux.comm_c, + t_aux, + challenges, + layers, + ) + }) + .collect::>>>>() } + } - info!( - "read_synth_porep: {}, gen_synth_porep {}", - read_synth_proofs, gen_synth_proofs + fn prove_layers_generate( + graph: &StackedBucketGraph, + pub_inputs: &PublicInputs<::Domain, ::Domain>, + comm_c: ::Domain, + t_aux: &TemporaryAuxCache, + challenges: Vec, + layers: usize, + ) -> Result>> { + assert_eq!(t_aux.labels.len(), layers); + assert_eq!( + pub_inputs.tau.as_ref().expect("as_ref failure").comm_d, + t_aux.tree_d.as_ref().expect("failed to get tree_d").root() ); + let get_drg_parents_columns = |x: usize| -> Result>> { let base_degree = graph.base_graph().degree(); @@ -195,180 +236,156 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr .collect() }; - let vanilla_proofs = (0..partition_count) - .map(|k| { - trace!("proving partition {}/{}", k + 1, partition_count); + THREAD_POOL.scoped(|scope| { + // Stacked commitment specifics + challenges + .into_par_iter() + .enumerate() + .map(|(challenge_index, challenge)| { + trace!(" challenge {} ({})", challenge, challenge_index); + assert!(challenge < graph.size(), "Invalid challenge"); + assert!(challenge > 0, "Invalid challenge"); + + let comm_d_proof = t_aux + .tree_d + .as_ref() + .expect("failed to get tree_d") + .gen_proof(challenge)?; + + let comm_d_proof_inner = comm_d_proof.clone(); + let challenge_inner = challenge; + scope.execute(move || { + assert!(comm_d_proof_inner.validate(challenge_inner)); + }); + + // Stacked replica column openings + let rcp = { + let (c_x, drg_parents, exp_parents) = { + assert!(t_aux.tree_c.is_some()); + let tree_c = t_aux.tree_c.as_ref().expect("failed to get tree_c"); + assert_eq!(comm_c, tree_c.root()); + + // All labels in C_X + trace!(" c_x"); + let c_x = t_aux.column(challenge as u32)?.into_proof(tree_c)?; + + // All labels in the DRG parents. + trace!(" drg_parents"); + let drg_parents = get_drg_parents_columns(challenge)? + .into_iter() + .map(|column| column.into_proof(tree_c)) + .collect::>()?; + + // Labels for the expander parents + trace!(" exp_parents"); + let exp_parents = get_exp_parents_columns(challenge)? + .into_iter() + .map(|column| column.into_proof(tree_c)) + .collect::>()?; + + (c_x, drg_parents, exp_parents) + }; - // Derive the set of challenges we are proving over. - let challenges = pub_inputs.challenges(layer_challenges, graph_size, Some(k)); + ReplicaColumnProof { + c_x, + drg_parents, + exp_parents, + } + }; - THREAD_POOL.scoped(|scope| { - // Stacked commitment specifics - challenges - .into_par_iter() - .enumerate() - .map(|(challenge_index, challenge)| { - trace!(" challenge {} ({})", challenge, challenge_index); - assert!(challenge < graph.size(), "Invalid challenge"); - assert!(challenge > 0, "Invalid challenge"); - - let comm_d_proof = t_aux - .tree_d - .as_ref() - .expect("failed to get tree_d") - .gen_proof(challenge)?; - - let comm_d_proof_inner = comm_d_proof.clone(); - let challenge_inner = challenge; - scope.execute(move || { - assert!(comm_d_proof_inner.validate(challenge_inner)); - }); + // Final replica layer openings + trace!("final replica layer openings"); + let comm_r_last_proof = t_aux.tree_r_last.gen_cached_proof( + challenge, + Some(t_aux.tree_r_last_config_rows_to_discard), + )?; + + let comm_r_last_proof_inner = comm_r_last_proof.clone(); + scope.execute(move || { + debug_assert!(comm_r_last_proof_inner.validate(challenge)); + }); + + // Labeling Proofs Layer 1..l + let mut labeling_proofs = Vec::with_capacity(layers); + let mut encoding_proof = None; + + for layer in 1..=layers { + trace!(" encoding proof layer {}", layer,); + let parents_data: Vec<::Domain> = if layer == 1 { + let mut parents = vec![0; graph.base_graph().degree()]; + graph.base_parents(challenge, &mut parents)?; + + parents + .into_par_iter() + .map(|parent| t_aux.domain_node_at_layer(layer, parent)) + .collect::>()? + } else { + let mut parents = vec![0; graph.degree()]; + graph.parents(challenge, &mut parents)?; + let base_parents_count = graph.base_graph().degree(); + + parents + .into_par_iter() + .enumerate() + .map(|(i, parent)| { + if i < base_parents_count { + // parents data for base parents is from the current layer + t_aux.domain_node_at_layer(layer, parent) + } else { + // parents data for exp parents is from the previous layer + t_aux.domain_node_at_layer(layer - 1, parent) + } + }) + .collect::>()? + }; - // Stacked replica column openings - let rcp = { - let (c_x, drg_parents, exp_parents) = { - assert!(t_aux.tree_c.is_some()); - let tree_c = - t_aux.tree_c.as_ref().expect("failed to get tree_c"); - assert_eq!(p_aux.comm_c, tree_c.root()); - - // All labels in C_X - trace!(" c_x"); - let c_x = t_aux.column(challenge as u32)?.into_proof(tree_c)?; - - // All labels in the DRG parents. - trace!(" drg_parents"); - let drg_parents = get_drg_parents_columns(challenge)? - .into_iter() - .map(|column| column.into_proof(tree_c)) - .collect::>()?; - - // Labels for the expander parents - trace!(" exp_parents"); - let exp_parents = get_exp_parents_columns(challenge)? - .into_iter() - .map(|column| column.into_proof(tree_c)) - .collect::>()?; - - (c_x, drg_parents, exp_parents) - }; - - ReplicaColumnProof { - c_x, - drg_parents, - exp_parents, - } - }; + // repeat parents + let mut parents_data_full = vec![Default::default(); TOTAL_PARENTS]; + for chunk in parents_data_full.chunks_mut(parents_data.len()) { + chunk.copy_from_slice(&parents_data[..chunk.len()]); + } - // Final replica layer openings - trace!("final replica layer openings"); - let comm_r_last_proof = t_aux.tree_r_last.gen_cached_proof( - challenge, - Some(t_aux.tree_r_last_config_rows_to_discard), - )?; + let proof = LabelingProof::::new( + layer as u32, + challenge as u64, + parents_data_full.clone(), + ); - let comm_r_last_proof_inner = comm_r_last_proof.clone(); + { + let labeled_node = *rcp.c_x.get_node_at_layer(layer)?; + let replica_id = &pub_inputs.replica_id; + let proof_inner = proof.clone(); scope.execute(move || { - debug_assert!(comm_r_last_proof_inner.validate(challenge)); - }); - - // Labeling Proofs Layer 1..l - let mut labeling_proofs = Vec::with_capacity(layers); - let mut encoding_proof = None; - - for layer in 1..=layers { - trace!(" encoding proof layer {}", layer,); - let parents_data: Vec<::Domain> = - if layer == 1 { - let mut parents = vec![0; graph.base_graph().degree()]; - graph.base_parents(challenge, &mut parents)?; - - parents - .into_par_iter() - .map(|parent| t_aux.domain_node_at_layer(layer, parent)) - .collect::>()? - } else { - let mut parents = vec![0; graph.degree()]; - graph.parents(challenge, &mut parents)?; - let base_parents_count = graph.base_graph().degree(); - - parents - .into_par_iter() - .enumerate() - .map(|(i, parent)| { - if i < base_parents_count { - // parents data for base parents is from the current layer - t_aux.domain_node_at_layer(layer, parent) - } else { - // parents data for exp parents is from the previous layer - t_aux.domain_node_at_layer(layer - 1, parent) - } - }) - .collect::>()? - }; - - // repeat parents - let mut parents_data_full = vec![Default::default(); TOTAL_PARENTS]; - for chunk in parents_data_full.chunks_mut(parents_data.len()) { - chunk.copy_from_slice(&parents_data[..chunk.len()]); - } - - let proof = LabelingProof::::new( - layer as u32, - challenge as u64, - parents_data_full.clone(), + assert!( + proof_inner.verify(replica_id, &labeled_node), + "Invalid encoding proof generated at layer {}", + layer, ); + trace!("Valid encoding proof generated at layer {}", layer); + }); + } - { - let labeled_node = *rcp.c_x.get_node_at_layer(layer)?; - let replica_id = &pub_inputs.replica_id; - let proof_inner = proof.clone(); - scope.execute(move || { - assert!( - proof_inner.verify(replica_id, &labeled_node), - "Invalid encoding proof generated at layer {}", - layer, - ); - trace!("Valid encoding proof generated at layer {}", layer); - }); - } - - labeling_proofs.push(proof); + labeling_proofs.push(proof); - if layer == layers { - encoding_proof = Some(EncodingProof::new( - layer as u32, - challenge as u64, - parents_data_full, - )); - } - } + if layer == layers { + encoding_proof = Some(EncodingProof::new( + layer as u32, + challenge as u64, + parents_data_full, + )); + } + } - Ok(Proof { - comm_d_proofs: comm_d_proof, - replica_column_proofs: rcp, - comm_r_last_proof, - labeling_proofs, - encoding_proof: encoding_proof.expect("invalid tapering"), - }) - }) - .collect() + Ok(Proof { + comm_d_proofs: comm_d_proof, + replica_column_proofs: rcp, + comm_r_last_proof, + labeling_proofs, + encoding_proof: encoding_proof.expect("invalid tapering"), + }) }) - }) - .collect::>>>>()?; - - // If synthetic vanilla proofs were generated, persist them here. - if gen_synth_proofs { - assert!( - vanilla_proofs.iter().skip(1).all(Vec::is_empty), - "synthetic proofs should be generated in a single partition", - ); - let synth_proofs = &vanilla_proofs[0]; - Self::write_synth_proofs(synth_proofs, pub_inputs, graph, layer_challenges, t_aux)?; - return Ok(vec![vec![]; partition_count]); - } - - Ok(vanilla_proofs) + .collect() + }) } fn write_synth_proofs( @@ -376,7 +393,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr pub_inputs: &PublicInputs<::Domain, ::Domain>, graph: &StackedBucketGraph, layer_challenges: &LayerChallenges, - t_aux: &TemporaryAuxCache, + path: PathBuf, ) -> Result<()> { use crate::stacked::vanilla::SynthChallenges; @@ -414,7 +431,6 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr } }); - let path = t_aux.synth_proofs_path(); info!("writing synth-porep vanilla proofs to file: {:?}", path); let file = File::create(&path).map(BufWriter::new).with_context(|| { format!( @@ -439,7 +455,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr sector_nodes: usize, pub_inputs: &PublicInputs<::Domain, ::Domain>, layer_challenges: &LayerChallenges, - t_aux: &TemporaryAuxCache, + path: PathBuf, partition_count: usize, ) -> Result>>> { ensure!( @@ -460,7 +476,6 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher> StackedDrg<'a, Tr .as_ref() .map(|tau| &tau.comm_r) .expect("unwrapping should not fail"); - let path = t_aux.synth_proofs_path(); info!("reading synthetic vanilla proofs from file: {:?}", path); let num_layers = layer_challenges.layers();