avoid locking and global caches

storage-proofs/porep/Cargo.toml

@@ -27,7 +27,6 @@ log = "0.4.7"
 pretty_assertions = "0.6.1"
 generic-array = "0.13.2"
 anyhow = "1.0.23"
-once_cell = "1.3.1"
 neptune = { version = "1.0.1", features = ["gpu"] }
 num_cpus = "1.10.1"
 hex = "0.4.2"

storage-proofs/porep/src/stacked/circuit/create_label.rs

@@ -167,7 +167,7 @@ mod tests {
         assert_eq!(cs.num_constraints(), 532_025);
 
         let (l1, l2) = data.split_at_mut(size * NODE_SIZE);
-        create_label_exp(&graph, &id_fr.into(), &*l2, l1, layer, node).unwrap();
+        create_label_exp(&graph, None, &id_fr.into(), &*l2, l1, layer, node).unwrap();
         let expected_raw = data_at_node(&l1, node).unwrap();
         let expected = bytes_into_fr(expected_raw).unwrap();
 

storage-proofs/porep/src/stacked/vanilla/cache.rs

@@ -1,5 +1,4 @@
 use std::path::PathBuf;
-use std::sync::RwLock;
 
 use anyhow::{bail, ensure, Context};
 use byteorder::{BigEndian, ByteOrder};
@@ -30,7 +29,7 @@ pub struct ParentCache {
     path: PathBuf,
     /// The total number of cache entries.
     num_cache_entries: u32,
-    cache: RwLock<CacheData>,
+    cache: CacheData,
 }
 
 #[derive(Debug)]
@@ -153,7 +152,7 @@ impl ParentCache {
         info!("parent cache: opened");
 
         Ok(ParentCache {
-            cache: RwLock::new(cache),
+            cache,
             path,
             num_cache_entries: cache_entries,
         })
@@ -211,40 +210,38 @@ impl ParentCache {
         info!("parent cache: written to disk");
 
         Ok(ParentCache {
-            cache: RwLock::new(CacheData::open(0, len, &path)?),
+            cache: CacheData::open(0, len, &path)?,
             path,
             num_cache_entries: cache_entries,
         })
     }
 
     /// Read a single cache element at position `node`.
-    pub fn read(&self, node: u32) -> Result<[u32; DEGREE]> {
-        let cache = self.cache.read().unwrap();
-        if cache.contains(node) {
-            return Ok(cache.read(node));
+    pub fn read(&mut self, node: u32) -> Result<[u32; DEGREE]> {
+        if self.cache.contains(node) {
+            return Ok(self.cache.read(node));
         }
 
         // not in memory, shift cache
-        drop(cache);
-        let cache = &mut *self.cache.write().unwrap();
         ensure!(
-            node >= cache.offset + cache.len,
+            node >= self.cache.offset + self.cache.len,
             "cache must be read in ascending order {} < {} + {}",
             node,
-            cache.offset,
-            cache.len,
+            self.cache.offset,
+            self.cache.len,
         );
 
         // Shift cache by its current size.
-        let new_offset = (self.num_cache_entries - cache.len).min(cache.offset + cache.len);
-        cache.shift(new_offset)?;
+        let new_offset =
+            (self.num_cache_entries - self.cache.len).min(self.cache.offset + self.cache.len);
+        self.cache.shift(new_offset)?;
 
-        Ok(cache.read(node))
+        Ok(self.cache.read(node))
     }
 
     /// Resets the partial cache to the beginning.
-    pub fn reset(&self) -> Result<()> {
-        self.cache.write().unwrap().reset()
+    pub fn reset(&mut self) -> Result<()> {
+        self.cache.reset()
     }
 }
 
@@ -284,7 +281,7 @@ mod tests {
         )
         .unwrap();
 
-        let cache = ParentCache::new(nodes, nodes, &graph).unwrap();
+        let mut cache = ParentCache::new(nodes, nodes, &graph).unwrap();
 
         for node in 0..nodes {
             let mut expected_parents = [0; DEGREE];
@@ -306,8 +303,8 @@ mod tests {
         )
         .unwrap();
 
-        let half_cache = ParentCache::new(nodes / 2, nodes, &graph).unwrap();
-        let quarter_cache = ParentCache::new(nodes / 4, nodes, &graph).unwrap();
+        let mut half_cache = ParentCache::new(nodes / 2, nodes, &graph).unwrap();
+        let mut quarter_cache = ParentCache::new(nodes / 4, nodes, &graph).unwrap();
 
         for node in 0..nodes {
             let mut expected_parents = [0; DEGREE];
@@ -321,11 +318,11 @@ mod tests {
 
             // some internal checks to make sure the cache works as expected
             assert_eq!(
-                half_cache.cache.read().unwrap().data.len() / DEGREE / NODE_BYTES,
+                half_cache.cache.data.len() / DEGREE / NODE_BYTES,
                 nodes as usize / 2
             );
             assert_eq!(
-                quarter_cache.cache.read().unwrap().data.len() / DEGREE / NODE_BYTES,
+                quarter_cache.cache.data.len() / DEGREE / NODE_BYTES,
                 nodes as usize / 4
             );
         }

storage-proofs/porep/src/stacked/vanilla/create_label.rs

@@ -10,10 +10,11 @@ use storage_proofs_core::{
     util::{data_at_node_offset, NODE_SIZE},
 };
 
-use super::graph::StackedBucketGraph;
+use super::{cache::ParentCache, graph::StackedBucketGraph};
 
 pub fn create_label<H: Hasher>(
     graph: &StackedBucketGraph<H>,
+    cache: Option<&mut ParentCache>,
     replica_id: &H::Domain,
     layer_labels: &mut [u8],
     layer_index: usize,
@@ -34,7 +35,7 @@ pub fn create_label<H: Hasher>(
             _mm_prefetch(prev.as_ptr() as *const i8, _MM_HINT_T0);
         }
 
-        graph.copy_parents_data(node as u32, &*layer_labels, hasher)?
+        graph.copy_parents_data(node as u32, &*layer_labels, hasher, cache)?
     } else {
         hasher.finish()
     };
@@ -52,6 +53,7 @@ pub fn create_label<H: Hasher>(
 
 pub fn create_label_exp<H: Hasher>(
     graph: &StackedBucketGraph<H>,
+    cache: Option<&mut ParentCache>,
     replica_id: &H::Domain,
     exp_parents_data: &[u8],
     layer_labels: &mut [u8],
@@ -73,7 +75,7 @@ pub fn create_label_exp<H: Hasher>(
             _mm_prefetch(prev.as_ptr() as *const i8, _MM_HINT_T0);
         }
 
-        graph.copy_parents_data_exp(node as u32, &*layer_labels, exp_parents_data, hasher)?
+        graph.copy_parents_data_exp(node as u32, &*layer_labels, exp_parents_data, hasher, cache)?
     } else {
         hasher.finish()
     };

storage-proofs/porep/src/stacked/vanilla/graph.rs

@@ -8,7 +8,6 @@ use std::arch::x86_64::*;
 
 use anyhow::ensure;
 use log::info;
-use once_cell::sync::OnceCell;
 use sha2raw::Sha256;
 use storage_proofs_core::{
     crypto::{
@@ -21,7 +20,6 @@ use storage_proofs_core::{
     error::Result,
     hasher::Hasher,
     parameter_cache::ParameterSetMetadata,
-    settings,
     util::NODE_SIZE,
 };
 
@@ -43,7 +41,6 @@ where
     feistel_keys: [feistel::Index; 4],
     feistel_precomputed: FeistelPrecomputed,
     id: String,
-    cache: Option<&'static ParentCache>,
     _h: PhantomData<H>,
 }
 
@@ -58,7 +55,6 @@ where
             .field("base_graph", &self.base_graph)
             .field("feistel_precomputed", &self.feistel_precomputed)
             .field("id", &self.id)
-            .field("cache", &self.cache)
             .finish()
     }
 }
@@ -100,8 +96,6 @@ where
         assert_eq!(expansion_degree, EXP_DEGREE);
         ensure!(nodes <= std::u32::MAX as usize, "too many nodes");
 
-        let use_cache = settings::SETTINGS.lock().unwrap().maximize_caching;
-
         let base_graph = match base_graph {
             Some(graph) => graph,
             None => G::new(nodes, base_degree, 0, porep_id)?,
@@ -115,68 +109,30 @@ where
         feistel_keys[2] = u64::from_le_bytes(raw_seed[16..24].try_into().unwrap());
         feistel_keys[3] = u64::from_le_bytes(raw_seed[24..32].try_into().unwrap());
 
-        let mut res = StackedGraph {
+        let res = StackedGraph {
             base_graph,
             id: format!(
                 "stacked_graph::StackedGraph{{expansion_degree: {} base_graph: {} }}",
                 expansion_degree, bg_id,
             ),
             expansion_degree,
-            cache: None,
             feistel_keys,
             feistel_precomputed: feistel::precompute((expansion_degree * nodes) as feistel::Index),
             _h: PhantomData,
         };
 
-        if use_cache {
-            info!("using parents cache");
-            res.parent_cache(nodes as u32)?;
-        }
-
         Ok(res)
     }
 
-    /// Resets the parent cache if one exists.
-    pub fn reset_parent_cache(&self) -> Result<()> {
-        if let Some(ref cache) = self.cache {
-            return cache.reset();
-        }
-
-        Ok(())
-    }
-
-    /// Returns a reference to the parent cache, initializing it lazily the first time this is called.
-    fn parent_cache(&mut self, cache_entries: u32) -> Result<()> {
-        const NODE_GIB: u32 = (1024 * 1024 * 1024) / NODE_SIZE as u32;
-
+    /// Returns a reference to the parent cache.
+    pub fn parent_cache(&self) -> Result<ParentCache> {
         // Number of nodes to be cached in memory
         const DEFAULT_CACHE_SIZE: u32 = 2048;
+        let cache_entries = self.size() as u32;
 
-        static INSTANCE_32_GIB: OnceCell<ParentCache> = OnceCell::new();
-        static INSTANCE_64_GIB: OnceCell<ParentCache> = OnceCell::new();
-
-        ensure!(
-            ((cache_entries == 32 * NODE_GIB) || (cache_entries == 64 * NODE_GIB)),
-            "Cache is only available for 32GiB and 64GiB sectors"
-        );
         info!("using parent_cache[{}]", cache_entries);
 
-        let cache = if cache_entries == 32 * NODE_GIB {
-            INSTANCE_32_GIB.get_or_init(|| {
-                ParentCache::new(DEFAULT_CACHE_SIZE, cache_entries, self)
-                    .expect("failed to fill 32GiB cache")
-            })
-        } else {
-            INSTANCE_64_GIB.get_or_init(|| {
-                ParentCache::new(DEFAULT_CACHE_SIZE, cache_entries, self)
-                    .expect("failed to fill 64GiB cache")
-            })
-        };
-
-        cache.reset()?;
-        self.cache = Some(cache);
-
-        Ok(())
+        ParentCache::new(DEFAULT_CACHE_SIZE, cache_entries, self)
     }
 
     pub fn copy_parents_data_exp(
@@ -185,8 +141,9 @@ where
         base_data: &[u8],
         exp_data: &[u8],
         hasher: Sha256,
+        mut cache: Option<&mut ParentCache>,
     ) -> Result<[u8; 32]> {
-        if let Some(cache) = self.cache {
+        if let Some(ref mut cache) = cache {
             let cache_parents = cache.read(node as u32)?;
             Ok(self.copy_parents_data_inner_exp(&cache_parents, base_data, exp_data, hasher))
         } else {
@@ -202,8 +159,9 @@ where
         node: u32,
         base_data: &[u8],
         hasher: Sha256,
+        mut cache: Option<&mut ParentCache>,
     ) -> Result<[u8; 32]> {
-        if let Some(cache) = self.cache {
+        if let Some(ref mut cache) = cache {
             let cache_parents = cache.read(node as u32)?;
             Ok(self.copy_parents_data_inner(&cache_parents, base_data, hasher))
         } else {
@@ -325,20 +283,15 @@ where
 
     #[inline]
     fn parents(&self, node: usize, parents: &mut [u32]) -> Result<()> {
-        if let Some(cache) = self.cache {
-            // Read from the cache
-            let cache_parents = cache.read(node as u32)?;
-            parents.copy_from_slice(&cache_parents);
-        } else {
-            self.base_parents(node, &mut parents[..self.base_graph().degree()])?;
-
-            // expanded_parents takes raw_node
-            self.expanded_parents(
-                node,
-                &mut parents[self.base_graph().degree()
-                    ..self.base_graph().degree() + self.expansion_degree()],
-            )?;
-        }
+        self.base_parents(node, &mut parents[..self.base_graph().degree()])?;
+
+        // expanded_parents takes raw_node
+        self.expanded_parents(
+            node,
+            &mut parents
+                [self.base_graph().degree()..self.base_graph().degree() + self.expansion_degree()],
+        )?;
+
         Ok(())
     }
 
@@ -438,31 +391,17 @@ where
     }
 
     pub fn base_parents(&self, node: usize, parents: &mut [u32]) -> Result<()> {
-        if let Some(cache) = self.cache {
-            // Read from the cache
-            let cache_parents = cache.read(node as u32)?;
-            parents.copy_from_slice(&cache_parents[..self.base_graph().degree()]);
-            Ok(())
-        } else {
-            // No cache usage, generate on demand.
-            self.base_graph().parents(node, parents)
-        }
+        // No cache usage, generate on demand.
+        self.base_graph().parents(node, parents)
     }
 
     /// Assign `self.expansion_degree` parents to `node` using an invertible permutation
     /// that is applied one way for the forward layers and one way for the reversed
     /// ones.
     #[inline]
     pub fn expanded_parents(&self, node: usize, parents: &mut [u32]) -> Result<()> {
-        if let Some(cache) = self.cache {
-            // Read from the cache
-            let cache_parents = cache.read(node as u32)?;
-            parents.copy_from_slice(&cache_parents[self.base_graph().degree()..]);
-        } else {
-            // No cache usage, generate on demand.
-            self.generate_expanded_parents(node, parents);
-        }
-
+        // No cache usage, generate on demand.
+        self.generate_expanded_parents(node, parents);
         Ok(())
     }
 }