feat(storage-proofs): partial caching for SDR (#1163)

* feat(storage-proofs): store sdr cache on disk and memmap it * refactor to allow remapping * add tests and shift cache properly * ensure that the cache is reset for every layer * add missing import * fix offset generation * avoid locking and global caches * fixup benches * default maximize_cache to true * ensure partial cache size is at most the full cache size * consistent use of porep_id and endianess * feat: make parents cache size configurable
filecoin-project · Jun 17, 2020 · 28ebe44 · 28ebe44
1 parent 332d0a9
commit 28ebe44
Show file tree

Hide file tree

Showing 12 changed files with 443 additions and 151 deletions.
diff --git a/fil-proofs-tooling/Cargo.toml b/fil-proofs-tooling/Cargo.toml
@@ -31,7 +31,7 @@ storage-proofs = { path = "../storage-proofs"}
 filecoin-proofs = { path = "../filecoin-proofs"}
 tempfile = "3.0.8"
 cpu-time = "1.0.0"
-git2 = "0.10.1"
+git2 = "0.13"
 heim = "0.0.9"
 futures-preview = "0.3.0-alpha.17"
 raw-cpuid = "7.0.3"

diff --git a/storage-proofs/core/src/parameter_cache.rs b/storage-proofs/core/src/parameter_cache.rs
@@ -134,7 +134,7 @@ fn ensure_ancestor_dirs_exist(cache_entry_path: PathBuf) -> Result<PathBuf> {
     Ok(cache_entry_path)
 }
 
-pub trait ParameterSetMetadata: Clone {
+pub trait ParameterSetMetadata {
     fn identifier(&self) -> String;
     fn sector_size(&self) -> u64;
 }

diff --git a/storage-proofs/core/src/settings.rs b/storage-proofs/core/src/settings.rs
@@ -22,19 +22,21 @@ pub struct Settings {
     pub use_gpu_tree_builder: bool,
     pub max_gpu_tree_batch_size: u32,
     pub rows_to_discard: u32,
+    pub sdr_parents_cache_size: u32,
 }
 
 impl Default for Settings {
     fn default() -> Self {
         Settings {
-            maximize_caching: false,
+            maximize_caching: true,
             pedersen_hash_exp_window_size: 16,
             use_gpu_column_builder: false,
             max_gpu_column_batch_size: 400_000,
             column_write_batch_size: 262_144,
             use_gpu_tree_builder: false,
             max_gpu_tree_batch_size: 700_000,
             rows_to_discard: 2,
+            sdr_parents_cache_size: 2048,
         }
     }
 }

diff --git a/storage-proofs/porep/Cargo.toml b/storage-proofs/porep/Cargo.toml
@@ -27,9 +27,10 @@ log = "0.4.7"
 pretty_assertions = "0.6.1"
 generic-array = "0.13.2"
 anyhow = "1.0.23"
-once_cell = "1.3.1"
 neptune = { version = "1.0.1", features = ["gpu"] }
 num_cpus = "1.10.1"
+hex = "0.4.2"
+byteorder = "1.3.4"
 
 [dev-dependencies]
 tempdir = "0.3.7"

diff --git a/storage-proofs/porep/benches/encode.rs b/storage-proofs/porep/benches/encode.rs
@@ -53,15 +53,25 @@ fn kdf_benchmark(c: &mut Criterion) {
         let graph = &graph;
         let replica_id = replica_id.clone();
 
-        b.iter(|| black_box(create_label_exp(graph, &replica_id, &*exp_data, data, 1, 2)))
+        b.iter(|| {
+            black_box(create_label_exp(
+                graph,
+                None,
+                &replica_id,
+                &*exp_data,
+                data,
+                1,
+                2,
+            ))
+        })
     });
 
     group.bench_function("non-exp", |b| {
         let mut data = data.clone();
         let graph = &graph;
         let replica_id = replica_id.clone();
 
-        b.iter(|| black_box(create_label(graph, &replica_id, &mut data, 1, 2)))
+        b.iter(|| black_box(create_label(graph, None, &replica_id, &mut data, 1, 2)))
     });
 
     group.finish();

diff --git a/storage-proofs/porep/src/stacked/circuit/create_label.rs b/storage-proofs/porep/src/stacked/circuit/create_label.rs
@@ -167,7 +167,7 @@ mod tests {
         assert_eq!(cs.num_constraints(), 532_025);
 
         let (l1, l2) = data.split_at_mut(size * NODE_SIZE);
-        create_label_exp(&graph, &id_fr.into(), &*l2, l1, layer, node).unwrap();
+        create_label_exp(&graph, None, &id_fr.into(), &*l2, l1, layer, node).unwrap();
         let expected_raw = data_at_node(&l1, node).unwrap();
         let expected = bytes_into_fr(expected_raw).unwrap();
 

diff --git a/storage-proofs/porep/src/stacked/circuit/proof.rs b/storage-proofs/porep/src/stacked/circuit/proof.rs
@@ -239,7 +239,7 @@ impl<'a, Tree: 'static + MerkleTreeTrait, G: 'static + Hasher>
 
             // exp parents
             let mut exp_parents = vec![0; graph.expansion_degree()];
-            graph.expanded_parents(challenge, &mut exp_parents);
+            graph.expanded_parents(challenge, &mut exp_parents)?;
 
             // Inclusion Proofs: expander parent node in comm_c
             for parent in exp_parents.into_iter() {