From df6e47e57609cb9355cd2849976d0a5d29ff4b3b Mon Sep 17 00:00:00 2001 From: Bernhard Schuster Date: Mon, 4 May 2020 18:40:47 +0200 Subject: [PATCH] step by step --- Cargo.lock | 4 + primitives/session/Cargo.toml | 6 + primitives/session/src/lib.rs | 2 + .../{trie => session}/src/offchain_hashdb.rs | 151 ++++++++++-------- primitives/trie/Cargo.toml | 2 - primitives/trie/src/lib.rs | 1 - 6 files changed, 96 insertions(+), 70 deletions(-) rename primitives/{trie => session}/src/offchain_hashdb.rs (59%) diff --git a/Cargo.lock b/Cargo.lock index f2296d442ed6e..03d801366bd8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7734,10 +7734,14 @@ dependencies = [ name = "sp-session" version = "2.0.0-dev" dependencies = [ + "hash-db", + "parity-scale-codec", "sp-api", "sp-core", + "sp-io", "sp-runtime", "sp-std", + "trie-db", ] [[package]] diff --git a/primitives/session/Cargo.toml b/primitives/session/Cargo.toml index 6d210b341f43b..ddd2123fff5a5 100644 --- a/primitives/session/Cargo.toml +++ b/primitives/session/Cargo.toml @@ -17,6 +17,12 @@ sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" } sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } sp-runtime = { version = "2.0.0-dev", optional = true, path = "../runtime" } +# x +codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false } +hash-db = { version = "0.15.2", default-features = false } +trie-db = { version = "0.20.1", default-features = false } +sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-dev"} + [features] default = [ "std" ] std = [ "sp-api/std", "sp-std/std", "sp-runtime", "sp-core/std" ] diff --git a/primitives/session/src/lib.rs b/primitives/session/src/lib.rs index 8e2a68d050983..f585cbe9c3d74 100644 --- a/primitives/session/src/lib.rs +++ b/primitives/session/src/lib.rs @@ -20,6 +20,8 @@ use sp_std::vec::Vec; +mod offchain_hashdb; + #[cfg(feature = "std")] use sp_runtime::{generic::BlockId, traits::Block as BlockT}; #[cfg(feature = "std")] diff --git a/primitives/trie/src/offchain_hashdb.rs b/primitives/session/src/offchain_hashdb.rs similarity index 59% rename from primitives/trie/src/offchain_hashdb.rs rename to primitives/session/src/offchain_hashdb.rs index db495f8f03be0..10cdc956bda9c 100644 --- a/primitives/trie/src/offchain_hashdb.rs +++ b/primitives/session/src/offchain_hashdb.rs @@ -22,38 +22,21 @@ use trie_db::node::{decode_hash, Node as EncodedNode, NodeHandle as EncodedNodeH use trie_db::{node::NodeKey, DBValue}; use trie_db::{CError, Result, TrieError, TrieHash, TrieLayout, TrieMut}; -use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX}; -use hashbrown::HashSet; - -use crate::nibble::{nibble_ops, BackingByteVec, NibbleSlice, NibbleVec}; -use crate::node_codec::NodeCodec; -use crate::rstd::{ - boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque, -}; - -use sp_core::offchain; - -use sp_staking::SessionIndex; -use sp_offchain::storage::Offchain; -use sp_offchain::storage::StorageKind; -use sp_offchain::StoragePrefix; -use trie_db::DBValue; -use codec::{Encode, Decode}; - -#[cfg(feature = "std")] -use log::trace; - -#[cfg(feature = "std")] -use crate::rstd::fmt::{self, Debug}; +use hash_db::{AsHashDB, HashDB, Hasher, Prefix, EMPTY_PREFIX}; +use core::marker::Sync; +use sp_core::offchain::{StorageKind,OffchainStorage}; +use sp_io::offchain; +use codec::{Encode, Decode}; +use sp_std::prelude::*; #[derive(Default)] pub struct AlternativeDB where - L: TrieLayout, + L: TrieLayout + Sync, { - session: SessionIndex, - _phantom: std::marker::PhantomData, + session_index: Vec, + _phantom: core::marker::PhantomData, } @@ -64,55 +47,74 @@ where /// session, tree_prefix -> [key,key,key,..] impl AlternativeDB where - L: TrieLayout, + L: TrieLayout + Sync, { pub fn prune(&self, key: &[u8]) { } /// set the session index for which to query the DB - pub fn set_session(&mut self, session: SessionIndex) { - self.session = session; + pub fn set_session(&mut self, session_index: E) { + self.session_index = session_index.encode(); } fn add_to_indices(&self, key: &[u8], tree_prefix: &[u8]) { - let mut mapping : Vec<(Vec,Vec)> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| { - ,Vec)> as Decode>::decode().unwrap(bytes.as_slice()) - }).ok().unwrap_or_else(|| Vec::with_capacity(1)); - mapping.push((key.to_vec(),value.to_vec())); + let mut mapping : Vec<(Vec,Vec)> = + offchain::local_storage_get( + StorageKind::PERSISTENT, + key) + .map(|bytes| { + ,Vec)> as Decode>::decode().unwrap(bytes.as_slice()) + }) + .ok() + .unwrap_or_else(|| { Vec::with_capacity(1) } ); + + mapping.push((key.to_vec(), tree_prefix.to_vec())); + let encoded = mapping.encode(); - offchain::local_storage_set(StorageKind::PERSISTENT, key, encoded.as_slice()); + offchain::local_storage_set( + StorageKind::PERSISTENT, + key, + encoded.as_slice()); } - fn get_index_with_session(&self, session_index: SessionIndex) -> Vec<(Vec,Vec)> { - let key: Vec = tracking_index(self.session_index); - let mut mapping : Vec<(Vec,Vec)> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| { + fn get_index_with_session(&self, session_index: &[u8]) -> Vec<(Vec,Vec)> { + let key: Vec = tracking_index(session_index); + let mapping : Vec<(Vec,Vec)> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| { ,Vec)> as Decode>::decode(bytes.as_slice()).unwrap() - }).ok().unwrap_or_else(|| { Vec::with_capacity(1)}) + }) + .ok() + .unwrap_or_else(|| { Vec::with_capacity(8) }); + mapping } /// Returns a vector of keys (which are vectors of bytes) - fn get_index_with_session_and_prefix(&self, session_index: SessionIndex, tree_prefix: &[u8]) -> Vec> { - let key: Vec = tracking_tracking_index_with_prefix(prefix, self.session_index); - let mut mapping : Vec> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| { + fn get_index_with_session_and_prefix(&self, session_index: &[u8], tree_prefix: &[u8]) -> Vec> { + let key: Vec = tracking_index_with_prefix(tree_prefix, session_index); + let mapping : Vec> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| { > as Decode>::decode(bytes.as_slice()).unwrap() - }).ok().unwrap_or_else(|| Vec::with_capacity(1)) + }).ok().unwrap_or_else(|| { + Vec::with_capacity(8) + }); + mapping } - fn set_index_with_session(&self, session_index: SessionIndex, val: Vec<(Vec,Vec)> ) { - let key: Vec = tracking_index(self.session_index); - offchain::local_storage_set(StorageKind::PERSISTENT, key, val.encode()); + fn set_index_with_session(&self, session_index: &[u8], val: Vec<(Vec,Vec)> ) { + let key: Vec = tracking_index(session_index); + let val = val.encode(); + offchain::local_storage_set(StorageKind::PERSISTENT, key, val.as_slice()); } /// Returns a vector of keys (which are vectors of bytes) - fn set_index_with_session_and_prefix(&self, session_index: SessionIndex, tree_prefix: &[u8], val: Vec>) { - let key: Vec = tracking_tracking_index_with_prefix(prefix, self.session_index); - offchain::local_storage_set(StorageKind::PERSISTENT, key, val.encode()); + fn set_index_with_session_and_prefix(&self, session_index: &[u8], tree_prefix: &[u8], val: Vec>) { + let key: Vec = tracking_index_with_prefix(tree_prefix, session_index); + let val = val.encode(); + offchain::local_storage_set(StorageKind::PERSISTENT, key, val.as_slice()); } - fn remove_from_indices(&self, key: &[u8], tree_prefix: &[u8], session_index: SessionIndex) { + fn remove_from_indices(&self, key: &[u8], tree_prefix: &[u8], session_index: &[u8]) { let mut index0 = self.index_with_session_and_prefix(session_index, tree_prefix); let index0 = index0 .into_iter() @@ -131,10 +133,9 @@ where } -const TRACKING_PREFIX : &'static [u8] = "__TRACKING__"; +const TRACKING_PREFIX : &'static [u8] = b"__TRACKING__"; -fn tracking_index_with_prefix(tree_prefix: Prefix, session_index: SessionIndex) -> Vec { - let session_index = session_index.to_be_bytes().as_slice(); +fn tracking_index_with_prefix(tree_prefix: Prefix, session_index: &[u8]) -> Vec { //@todo probably waaaaay to slow // _ + _ + _ + _ let total_len = TRACKING_PREFIX.len() + 1 + session_index.len() + 1 + tree_prefix.0.len() + 1 + tree_prefix.1.is_some() + 1 + 2; @@ -150,8 +151,7 @@ fn tracking_index_with_prefix(tree_prefix: Prefix, session_index: SessionIndex) } } -fn tracking_index(tree_prefix: Prefix, session_index: SessionIndex) -> Vec { - let session_index = session_index.to_be_bytes().as_slice(); +fn tracking_index(tree_prefix: Prefix, session_index: &[u8]) -> Vec { //@todo probably waaaaay to slow // _ + _ + _ + _ let total_len = TRACKING_PREFIX.len() + 1 + session_index.len() + 1 + tree_prefix.0.len() + 1 + tree_prefix.1.is_some() + 1 + 2; @@ -168,10 +168,10 @@ fn tracking_index(tree_prefix: Prefix, session_index: SessionIndex) -> Vec { //@todo define the extra prefix -const PREFIX: &'static [u8] = &b"TO_BE_DEFINED"; +const PREFIX: &'static [u8] = &b"TO_BE_DEFINED"[..]; /// Concatenate the static prefix with a tree prefix. -fn prefix_glue(key: &[u8], tree_prefix: Prefix, session_index: SessionIndex) -> Vec { +fn prefix_glue(key: &[u8], tree_prefix: Prefix, session_index: &[u8]) -> Vec { let session_index = session_index.to_be_bytes().as_slice(); //@todo probably waaaaay to slow // _ + _ + _ + _ @@ -192,37 +192,54 @@ fn prefix_glue(key: &[u8], tree_prefix: Prefix, session_index: SessionIndex) -> impl HashDB for AlternativeDB where - L: TrieLayout, + L: TrieLayout + Sync + Send, T: Encode, - <::Hash as Trait>::Out: AsRef + Default, //@todo verify this is true for all sane hashes such as H256 + <::Hash as Hasher>::Out: AsRef + Default, //@todo verify this is true for all sane hashes such as H256 { - fn get(&self, key: &<::Hash as Trait>::Out, prefix: Prefix) -> Option { - let key: Vec = prefix_glue(key.as_ref(), prefix, self.session_index); + fn get(&self, key: &<::Hash as Hasher>::Out, prefix: Prefix) -> Option { + let key: Vec = prefix_glue(key.as_ref(), prefix, self.session_index.as_slice()); offchain::local_storage_get(StorageKind::PERSISTENT, key).ok().flatten() } - fn contains(&self, key: &<::Hash as Trait>::Out, prefix: Prefix) -> bool { - let key: Vec = prefix_glue(key.as_ref(), prefix, self.session_index); + fn contains(&self, key: &<::Hash as Hasher>::Out, prefix: Prefix) -> bool { + let key: Vec = prefix_glue(key.as_ref(), prefix, self.session_index.as_slice()); offchain::local_storage_set(StorageKind::PERSISTENT, key).ok().flatten().is_some() } - fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out { + fn insert(&mut self, prefix: Prefix, value: &[u8]) -> <::Hash as Hasher>::Out { let hasher = L::Hash::default(); let digest = hasher.hash(value); - let key: Vec = prefix_glue(digest.as_ref(), prefix, self.session_index); + let key: Vec = prefix_glue(digest.as_ref(), prefix, self.session_index.as_slice()); offchain::local_storage_set(StorageKind::PERSISTENT, key, value); digest } - fn emplace(&mut self, key: <::Hash as Trait>::Out, prefix: Prefix, value: T) { - let key: Vec = prefix_glue(key.as_ref(), prefix); + fn emplace(&mut self, key: <::Hash as Hasher>::Out, prefix: Prefix, value: T) { + let key: Vec = prefix_glue(key.as_ref(), prefix, self.session_index.as_slice()); let value : Vec = value.encode(); self.insert(prefix, value.as_slice()); } - fn remove(&mut self, key: &<::Hash as Trait>::Out, prefix: Prefix) { + fn remove(&mut self, key: &<::Hash as Hasher>::Out, prefix: Prefix) { //@todo no API for that just yet + unimplemented!("can;t remove just yet") + } +} +impl AsHashDB for AlternativeDB +where + L: TrieLayout + Sync + Send, + T: Encode, + <::Hash as Hasher>::Out: AsRef + Default, +{ + /// Perform upcast to HashDB for anything that derives from HashDB. + fn as_hash_db(&self) -> &dyn HashDB<::Hash , T> { + self + } + /// Perform mutable upcast to HashDB for anything that derives from HashDB. + fn as_hash_db_mut<'a>(&'a mut self) -> &'a mut (dyn HashDB<::Hash , T> + 'a) { + self } + } diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index bb4bac2cd2b6b..c010b3262d7e2 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -24,8 +24,6 @@ trie-db = { version = "0.20.1", default-features = false } trie-root = { version = "0.16.0", default-features = false } memory-db = { version = "0.20.0", default-features = false } sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" } -# sp-offchain = { version = "2.0.0-dev", default-features = false, path = "../offchain" } -# sp-staking = { version = "2.0.0-dev", default-features = false, path = "../staking" } [dev-dependencies] trie-bench = "0.21.0" diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index dd5810de730de..f328b71750ba4 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -23,7 +23,6 @@ mod node_header; mod node_codec; mod storage_proof; mod trie_stream; -mod offchain_hashdb; use sp_std::boxed::Box; use sp_std::marker::PhantomData;