From 98314fbcfc9540a2ce94a784a90469b967950404 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 6 Sep 2021 19:53:23 +0200 Subject: [PATCH 1/3] refactor(storage-proofs-porep): use slice indexing instead of fake pointers --- .../src/stacked/vanilla/create_label/multi.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs b/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs index 92d332998..4dd60401f 100644 --- a/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs +++ b/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs @@ -57,7 +57,7 @@ const SHA256_INITIAL_DIGEST: [u32; 8] = [ fn fill_buffer( cur_node: u64, parents_cache: &CacheReader, - mut cur_parent: &[u32], // parents for this node + cur_parent: &[u32], // parents for this node layer_labels: &UnsafeSlice<'_, u32>, exp_labels: Option<&UnsafeSlice<'_, u32>>, // None for layer0 buf: &mut [u8], @@ -85,39 +85,32 @@ fn fill_buffer( // which we know is not ready and will be filled in the main loop for k in 0..BASE_DEGREE - 1 { unsafe { - if cur_parent[0] as u64 >= parents_cache.get_consumer() { + if cur_parent[k] as u64 >= parents_cache.get_consumer() { // Node is not ready base_parent_missing.set(k); } else { let parent_data = { - let offset = cur_parent[0] as usize * NODE_WORDS; + let offset = cur_parent[k] as usize * NODE_WORDS; &layer_labels.as_slice()[offset..offset + NODE_WORDS] }; let a = SHA_BLOCK_SIZE + (NODE_SIZE * k); buf[a..a + NODE_SIZE].copy_from_slice(parent_data.as_byte_slice()); }; - - // Advance pointer for the last base parent - cur_parent = &cur_parent[1..]; } } - // Advance pointer for the last base parent - cur_parent = &cur_parent[1..]; } else { base_parent_missing.set_upto(BASE_DEGREE as u8); - cur_parent = &cur_parent[BASE_DEGREE..]; } if let Some(exp_labels) = exp_labels { // Read from each of the expander parent nodes - for k in BASE_DEGREE..DEGREE { + for (k, parent) in cur_parent.iter().enumerate().take(DEGREE).skip(BASE_DEGREE) { let parent_data = unsafe { - let offset = cur_parent[0] as usize * NODE_WORDS; + let offset = *parent as usize * NODE_WORDS; &exp_labels.as_slice()[offset..offset + NODE_WORDS] }; let a = SHA_BLOCK_SIZE + (NODE_SIZE * k); buf[a..a + NODE_SIZE].copy_from_slice(parent_data.as_byte_slice()); - cur_parent = &cur_parent[1..]; } } } From ec39b4b55189996a692ed1bb4ed3fdea800431f7 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 6 Sep 2021 20:39:11 +0200 Subject: [PATCH 2/3] fix(storage-proofs-porep): select optimal parents in multicore sdr Based on https://github.com/filecoin-project/rust-fil-proofs/pull/1477 Thanks to @qy3u for finding --- .../src/stacked/vanilla/create_label/multi.rs | 25 ++++++++++++++----- .../src/stacked/vanilla/graph.rs | 5 ++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs b/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs index 4dd60401f..139ccbb0e 100644 --- a/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs +++ b/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs @@ -18,6 +18,7 @@ use generic_array::{ use log::{debug, info}; use mapr::MmapMut; use merkletree::store::{DiskStore, Store, StoreConfig}; +use storage_proofs_core::api_version::ApiVersion; use storage_proofs_core::{ cache_key::CacheKey, drgraph::{Graph, BASE_DEGREE}, @@ -54,6 +55,7 @@ const SHA256_INITIAL_DIGEST: [u32; 8] = [ ]; #[inline] +#[allow(clippy::too_many_arguments)] fn fill_buffer( cur_node: u64, parents_cache: &CacheReader, @@ -62,6 +64,7 @@ fn fill_buffer( exp_labels: Option<&UnsafeSlice<'_, u32>>, // None for layer0 buf: &mut [u8], base_parent_missing: &mut BitMask, + api_version: ApiVersion, ) { let cur_node_swap = cur_node.to_be_bytes(); // Note switch to big endian buf[36..44].copy_from_slice(&cur_node_swap); // update buf with current node @@ -74,16 +77,18 @@ fn fill_buffer( compress256!(cur_node_ptr, buf, 1); // Fill in the base parents - // Node 5 (prev node) will always be missing, and there tend to be - // frequent close references. + let (predecessor_index, other_drg_parents) = match api_version { + ApiVersion::V1_0_0 => (BASE_DEGREE - 1, (0..BASE_DEGREE - 1)), + ApiVersion::V1_1_0 => (0, (1..BASE_DEGREE)), + }; + if cur_node > MIN_BASE_PARENT_NODE { - // Mark base parent 5 as missing - // base_parent_missing.set_all(0x20); - base_parent_missing.set(5); + // Mark base parent predrecessor as missing + base_parent_missing.set(predecessor_index); // Skip the last base parent - it always points to the preceding node, // which we know is not ready and will be filled in the main loop - for k in 0..BASE_DEGREE - 1 { + for k in other_drg_parents { unsafe { if cur_parent[k] as u64 >= parents_cache.get_consumer() { // Node is not ready @@ -142,6 +147,7 @@ fn create_label_runner( lookahead: u64, ring_buf: &RingBuf, base_parent_missing: &UnsafeSlice<'_, BitMask>, + api_version: ApiVersion, ) { info!("created label runner"); // Label data bytes per node @@ -180,6 +186,7 @@ fn create_label_runner( exp_labels, buf, bpm, + api_version, ); } @@ -193,6 +200,7 @@ fn create_label_runner( } } +#[allow(clippy::too_many_arguments)] fn create_layer_labels( parents_cache: &CacheReader, replica_id: &[u8], @@ -201,6 +209,7 @@ fn create_layer_labels( num_nodes: u64, cur_layer: u32, core_group: Arc>>>, + api_version: ApiVersion, ) { info!("Creating labels for layer {}", cur_layer); // num_producers is the number of producer threads @@ -277,6 +286,7 @@ fn create_layer_labels( lookahead as u64, ring_buf, base_parent_missing, + api_version, ) })); } @@ -337,6 +347,7 @@ fn create_layer_labels( cur_node_ptr = &mut cur_node_ptr[8..]; // Grab the current slot of the ring_buf let buf = unsafe { ring_buf.slot_mut(cur_slot) }; + // Fill in the base parents for k in 0..BASE_DEGREE { let bpm = unsafe { base_parent_missing.get(cur_slot) }; @@ -492,6 +503,7 @@ pub fn create_labels_for_encoding ApiVersion { + self.api_version + } } impl ParameterSetMetadata for StackedGraph From 4af8e48c24b749b8ea56ae273a84dbd6ff071fb9 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 7 Sep 2021 20:11:40 +0200 Subject: [PATCH 3/3] fix(storage-proofs-porep): actually clear base_parents_missing --- .../src/stacked/vanilla/create_label/multi.rs | 3 ++- storage-proofs-porep/src/stacked/vanilla/utils.rs | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs b/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs index 139ccbb0e..6016da4f4 100644 --- a/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs +++ b/storage-proofs-porep/src/stacked/vanilla/create_label/multi.rs @@ -83,7 +83,8 @@ fn fill_buffer( }; if cur_node > MIN_BASE_PARENT_NODE { - // Mark base parent predrecessor as missing + // Mark base parent predecessor as missing + base_parent_missing.clear(); base_parent_missing.set(predecessor_index); // Skip the last base parent - it always points to the preceding node, diff --git a/storage-proofs-porep/src/stacked/vanilla/utils.rs b/storage-proofs-porep/src/stacked/vanilla/utils.rs index 3ce5c3a0e..a577003dc 100644 --- a/storage-proofs-porep/src/stacked/vanilla/utils.rs +++ b/storage-proofs-porep/src/stacked/vanilla/utils.rs @@ -84,6 +84,11 @@ impl BitMask { pub fn get(self, i: usize) -> bool { self.0 & (1 << i) != 0 } + + #[inline] + pub fn clear(&mut self) { + self.0 = 0; + } } #[derive(Debug)]