Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Commit

Permalink
step by step
Browse files Browse the repository at this point in the history
  • Loading branch information
Bernhard Schuster committed May 4, 2020
1 parent 0015c4d commit df6e47e
Show file tree
Hide file tree
Showing 6 changed files with 96 additions and 70 deletions.
4 changes: 4 additions & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 6 additions & 0 deletions primitives/session/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ sp-std = { version = "2.0.0-dev", default-features = false, path = "../std" }
sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" }
sp-runtime = { version = "2.0.0-dev", optional = true, path = "../runtime" }

# x
codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false }
hash-db = { version = "0.15.2", default-features = false }
trie-db = { version = "0.20.1", default-features = false }
sp-io ={ path = "../../primitives/io", default-features = false , version = "2.0.0-dev"}

[features]
default = [ "std" ]
std = [ "sp-api/std", "sp-std/std", "sp-runtime", "sp-core/std" ]
2 changes: 2 additions & 0 deletions primitives/session/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@

use sp_std::vec::Vec;

mod offchain_hashdb;

#[cfg(feature = "std")]
use sp_runtime::{generic::BlockId, traits::Block as BlockT};
#[cfg(feature = "std")]
Expand Down
151 changes: 84 additions & 67 deletions primitives/trie/src/offchain_hashdb.rs → primitives/session/src/offchain_hashdb.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,38 +22,21 @@ use trie_db::node::{decode_hash, Node as EncodedNode, NodeHandle as EncodedNodeH
use trie_db::{node::NodeKey, DBValue};
use trie_db::{CError, Result, TrieError, TrieHash, TrieLayout, TrieMut};

use hash_db::{HashDB, Hasher, Prefix, EMPTY_PREFIX};
use hashbrown::HashSet;

use crate::nibble::{nibble_ops, BackingByteVec, NibbleSlice, NibbleVec};
use crate::node_codec::NodeCodec;
use crate::rstd::{
boxed::Box, convert::TryFrom, hash::Hash, mem, ops::Index, result, vec::Vec, VecDeque,
};

use sp_core::offchain;

use sp_staking::SessionIndex;
use sp_offchain::storage::Offchain;
use sp_offchain::storage::StorageKind;
use sp_offchain::StoragePrefix;
use trie_db::DBValue;
use codec::{Encode, Decode};

#[cfg(feature = "std")]
use log::trace;

#[cfg(feature = "std")]
use crate::rstd::fmt::{self, Debug};
use hash_db::{AsHashDB, HashDB, Hasher, Prefix, EMPTY_PREFIX};
use core::marker::Sync;

use sp_core::offchain::{StorageKind,OffchainStorage};
use sp_io::offchain;
use codec::{Encode, Decode};
use sp_std::prelude::*;

#[derive(Default)]
pub struct AlternativeDB<L>
where
L: TrieLayout,
L: TrieLayout + Sync,
{
session: SessionIndex,
_phantom: std::marker::PhantomData<L>,
session_index: Vec<u8>,
_phantom: core::marker::PhantomData<L>,
}


Expand All @@ -64,55 +47,74 @@ where
/// session, tree_prefix -> [key,key,key,..]
impl<L> AlternativeDB<L>
where
L: TrieLayout,
L: TrieLayout + Sync,
{
pub fn prune(&self, key: &[u8]) {

}

/// set the session index for which to query the DB
pub fn set_session(&mut self, session: SessionIndex) {
self.session = session;
pub fn set_session<E: Encode>(&mut self, session_index: E) {
self.session_index = session_index.encode();
}


fn add_to_indices(&self, key: &[u8], tree_prefix: &[u8]) {
let mut mapping : Vec<(Vec<u8>,Vec<u8>)> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| {
<Vec<(Vec<u8>,Vec<u8>)> as Decode>::decode().unwrap(bytes.as_slice())
}).ok().unwrap_or_else(|| Vec::with_capacity(1));
mapping.push((key.to_vec(),value.to_vec()));
let mut mapping : Vec<(Vec<u8>,Vec<u8>)> =
offchain::local_storage_get(
StorageKind::PERSISTENT,
key)
.map(|bytes| {
<Vec<(Vec<u8>,Vec<u8>)> as Decode>::decode().unwrap(bytes.as_slice())
})
.ok()
.unwrap_or_else(|| { Vec::with_capacity(1) } );

mapping.push((key.to_vec(), tree_prefix.to_vec()));

let encoded = mapping.encode();
offchain::local_storage_set(StorageKind::PERSISTENT, key, encoded.as_slice());
offchain::local_storage_set(
StorageKind::PERSISTENT,
key,
encoded.as_slice());
}

fn get_index_with_session(&self, session_index: SessionIndex) -> Vec<(Vec<u8>,Vec<u8>)> {
let key: Vec<u8> = tracking_index(self.session_index);
let mut mapping : Vec<(Vec<u8>,Vec<u8>)> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| {
fn get_index_with_session(&self, session_index: &[u8]) -> Vec<(Vec<u8>,Vec<u8>)> {
let key: Vec<u8> = tracking_index(session_index);
let mapping : Vec<(Vec<u8>,Vec<u8>)> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| {
<Vec<(Vec<u8>,Vec<u8>)> as Decode>::decode(bytes.as_slice()).unwrap()
}).ok().unwrap_or_else(|| { Vec::with_capacity(1)})
})
.ok()
.unwrap_or_else(|| { Vec::with_capacity(8) });
mapping
}

/// Returns a vector of keys (which are vectors of bytes)
fn get_index_with_session_and_prefix(&self, session_index: SessionIndex, tree_prefix: &[u8]) -> Vec<Vec<u8>> {
let key: Vec<u8> = tracking_tracking_index_with_prefix(prefix, self.session_index);
let mut mapping : Vec<Vec<u8>> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| {
fn get_index_with_session_and_prefix(&self, session_index: &[u8], tree_prefix: &[u8]) -> Vec<Vec<u8>> {
let key: Vec<u8> = tracking_index_with_prefix(tree_prefix, session_index);
let mapping : Vec<Vec<u8>> = offchain::local_storage_get(StorageKind::PERSISTENT, key).map(|bytes| {
<Vec<Vec<u8>> as Decode>::decode(bytes.as_slice()).unwrap()
}).ok().unwrap_or_else(|| Vec::with_capacity(1))
}).ok().unwrap_or_else(|| {
Vec::with_capacity(8)
});
mapping
}


fn set_index_with_session(&self, session_index: SessionIndex, val: Vec<(Vec<u8>,Vec<u8>)> ) {
let key: Vec<u8> = tracking_index(self.session_index);
offchain::local_storage_set(StorageKind::PERSISTENT, key, val.encode());
fn set_index_with_session(&self, session_index: &[u8], val: Vec<(Vec<u8>,Vec<u8>)> ) {
let key: Vec<u8> = tracking_index(session_index);
let val = val.encode();
offchain::local_storage_set(StorageKind::PERSISTENT, key, val.as_slice());
}

/// Returns a vector of keys (which are vectors of bytes)
fn set_index_with_session_and_prefix(&self, session_index: SessionIndex, tree_prefix: &[u8], val: Vec<Vec<u8>>) {
let key: Vec<u8> = tracking_tracking_index_with_prefix(prefix, self.session_index);
offchain::local_storage_set(StorageKind::PERSISTENT, key, val.encode());
fn set_index_with_session_and_prefix(&self, session_index: &[u8], tree_prefix: &[u8], val: Vec<Vec<u8>>) {
let key: Vec<u8> = tracking_index_with_prefix(tree_prefix, session_index);
let val = val.encode();
offchain::local_storage_set(StorageKind::PERSISTENT, key, val.as_slice());
}

fn remove_from_indices(&self, key: &[u8], tree_prefix: &[u8], session_index: SessionIndex) {
fn remove_from_indices(&self, key: &[u8], tree_prefix: &[u8], session_index: &[u8]) {
let mut index0 = self.index_with_session_and_prefix(session_index, tree_prefix);
let index0 = index0
.into_iter()
Expand All @@ -131,10 +133,9 @@ where
}


const TRACKING_PREFIX : &'static [u8] = "__TRACKING__";
const TRACKING_PREFIX : &'static [u8] = b"__TRACKING__";

fn tracking_index_with_prefix(tree_prefix: Prefix, session_index: SessionIndex) -> Vec<u8> {
let session_index = session_index.to_be_bytes().as_slice();
fn tracking_index_with_prefix(tree_prefix: Prefix, session_index: &[u8]) -> Vec<u8> {
//@todo probably waaaaay to slow
// _ + _ + _ + _
let total_len = TRACKING_PREFIX.len() + 1 + session_index.len() + 1 + tree_prefix.0.len() + 1 + tree_prefix.1.is_some() + 1 + 2;
Expand All @@ -150,8 +151,7 @@ fn tracking_index_with_prefix(tree_prefix: Prefix, session_index: SessionIndex)
}
}

fn tracking_index(tree_prefix: Prefix, session_index: SessionIndex) -> Vec<u8> {
let session_index = session_index.to_be_bytes().as_slice();
fn tracking_index(tree_prefix: Prefix, session_index: &[u8]) -> Vec<u8> {
//@todo probably waaaaay to slow
// _ + _ + _ + _
let total_len = TRACKING_PREFIX.len() + 1 + session_index.len() + 1 + tree_prefix.0.len() + 1 + tree_prefix.1.is_some() + 1 + 2;
Expand All @@ -168,10 +168,10 @@ fn tracking_index(tree_prefix: Prefix, session_index: SessionIndex) -> Vec<u8> {


//@todo define the extra prefix
const PREFIX: &'static [u8] = &b"TO_BE_DEFINED";
const PREFIX: &'static [u8] = &b"TO_BE_DEFINED"[..];

/// Concatenate the static prefix with a tree prefix.
fn prefix_glue(key: &[u8], tree_prefix: Prefix, session_index: SessionIndex) -> Vec<u8> {
fn prefix_glue(key: &[u8], tree_prefix: Prefix, session_index: &[u8]) -> Vec<u8> {
let session_index = session_index.to_be_bytes().as_slice();
//@todo probably waaaaay to slow
// _ + _ + _ + _
Expand All @@ -192,37 +192,54 @@ fn prefix_glue(key: &[u8], tree_prefix: Prefix, session_index: SessionIndex) ->

impl<L,T> HashDB<L::Hash, T> for AlternativeDB<L>
where
L: TrieLayout,
L: TrieLayout + Sync + Send,
T: Encode,
<<L as trie_db::TrieLayout>::Hash as Trait>::Out: AsRef<u8> + Default, //@todo verify this is true for all sane hashes such as H256
<<L as trie_db::TrieLayout>::Hash as Hasher>::Out: AsRef<u8> + Default, //@todo verify this is true for all sane hashes such as H256
{

fn get(&self, key: &<<L as trie_db::TrieLayout>::Hash as Trait>::Out, prefix: Prefix) -> Option<T> {
let key: Vec<u8> = prefix_glue(key.as_ref(), prefix, self.session_index);
fn get(&self, key: &<<L as trie_db::TrieLayout>::Hash as Hasher>::Out, prefix: Prefix) -> Option<T> {
let key: Vec<u8> = prefix_glue(key.as_ref(), prefix, self.session_index.as_slice());
offchain::local_storage_get(StorageKind::PERSISTENT, key).ok().flatten()
}

fn contains(&self, key: &<<L as trie_db::TrieLayout>::Hash as Trait>::Out, prefix: Prefix) -> bool {
let key: Vec<u8> = prefix_glue(key.as_ref(), prefix, self.session_index);
fn contains(&self, key: &<<L as trie_db::TrieLayout>::Hash as Hasher>::Out, prefix: Prefix) -> bool {
let key: Vec<u8> = prefix_glue(key.as_ref(), prefix, self.session_index.as_slice());
offchain::local_storage_set(StorageKind::PERSISTENT, key).ok().flatten().is_some()
}

fn insert(&mut self, prefix: Prefix, value: &[u8]) -> H::Out {
fn insert(&mut self, prefix: Prefix, value: &[u8]) -> <<L as trie_db::TrieLayout>::Hash as Hasher>::Out {
let hasher = L::Hash::default();
let digest = hasher.hash(value);
let key: Vec<u8> = prefix_glue(digest.as_ref(), prefix, self.session_index);
let key: Vec<u8> = prefix_glue(digest.as_ref(), prefix, self.session_index.as_slice());
offchain::local_storage_set(StorageKind::PERSISTENT, key, value);
digest
}

fn emplace(&mut self, key: <<L as trie_db::TrieLayout>::Hash as Trait>::Out, prefix: Prefix, value: T) {
let key: Vec<u8> = prefix_glue(key.as_ref(), prefix);
fn emplace(&mut self, key: <<L as trie_db::TrieLayout>::Hash as Hasher>::Out, prefix: Prefix, value: T) {
let key: Vec<u8> = prefix_glue(key.as_ref(), prefix, self.session_index.as_slice());
let value : Vec<u8> = value.encode();
self.insert(prefix, value.as_slice());
}

fn remove(&mut self, key: &<<L as trie_db::TrieLayout>::Hash as Trait>::Out, prefix: Prefix) {
fn remove(&mut self, key: &<<L as trie_db::TrieLayout>::Hash as Hasher>::Out, prefix: Prefix) {
//@todo no API for that just yet
unimplemented!("can;t remove just yet")
}
}

impl<L,T> AsHashDB<L::Hash, T> for AlternativeDB<L>
where
L: TrieLayout + Sync + Send,
T: Encode,
<<L as trie_db::TrieLayout>::Hash as Hasher>::Out: AsRef<u8> + Default,
{
/// Perform upcast to HashDB for anything that derives from HashDB.
fn as_hash_db(&self) -> &dyn HashDB<<L as trie_db::TrieLayout>::Hash , T> {
self
}
/// Perform mutable upcast to HashDB for anything that derives from HashDB.
fn as_hash_db_mut<'a>(&'a mut self) -> &'a mut (dyn HashDB<<L as trie_db::TrieLayout>::Hash , T> + 'a) {
self
}

}
2 changes: 0 additions & 2 deletions primitives/trie/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ trie-db = { version = "0.20.1", default-features = false }
trie-root = { version = "0.16.0", default-features = false }
memory-db = { version = "0.20.0", default-features = false }
sp-core = { version = "2.0.0-dev", default-features = false, path = "../core" }
# sp-offchain = { version = "2.0.0-dev", default-features = false, path = "../offchain" }
# sp-staking = { version = "2.0.0-dev", default-features = false, path = "../staking" }

[dev-dependencies]
trie-bench = "0.21.0"
Expand Down
1 change: 0 additions & 1 deletion primitives/trie/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ mod node_header;
mod node_codec;
mod storage_proof;
mod trie_stream;
mod offchain_hashdb;

use sp_std::boxed::Box;
use sp_std::marker::PhantomData;
Expand Down

0 comments on commit df6e47e

Please sign in to comment.