From ef20f40f659dd4c85b201903984f282d8ae0ddfa Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 9 Jul 2019 15:05:58 +0200 Subject: [PATCH 001/129] init some bench --- Cargo.lock | 2 + core/state-machine/Cargo.toml | 10 ++ core/state-machine/benches/bench.rs | 119 ++++++++++++++++++++ core/state-machine/src/lib.rs | 2 + core/state-machine/src/overlayed_changes.rs | 12 ++ 5 files changed, 145 insertions(+) create mode 100644 core/state-machine/benches/bench.rs diff --git a/Cargo.lock b/Cargo.lock index 08cfbc63ae3a8..6fae0286b0b8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4679,12 +4679,14 @@ dependencies = [ name = "substrate-state-machine" version = "2.0.0" dependencies = [ + "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-codec 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-panic-handler 2.0.0", "substrate-primitives 2.0.0", "substrate-trie 2.0.0", diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index bcb471d5306db..670a777364e31 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -19,3 +19,13 @@ num-traits = "0.2" [dev-dependencies] hex-literal = "0.2.0" +criterion = "0.2" +rand = "0.6" + +[[bench]] +name = "bench" +harness = false + +[features] +default = ["bench"] +bench = [] diff --git a/core/state-machine/benches/bench.rs b/core/state-machine/benches/bench.rs new file mode 100644 index 0000000000000..ffae896c84251 --- /dev/null +++ b/core/state-machine/benches/bench.rs @@ -0,0 +1,119 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use criterion::{Criterion, black_box, Bencher}; +use criterion::{criterion_group, criterion_main}; +use substrate_state_machine::OverlayedChanges; +use substrate_state_machine::BenchOverlay; + +const CONTENT_KEY_SIZE: usize = 32; + +fn get_content(seed: u64, len: usize) -> Vec { + use rand::SeedableRng; + use rand::RngCore; + let mut rng = rand::rngs::SmallRng::seed_from_u64(seed); + let mut data = vec![0u8; len]; + rng.fill_bytes(&mut data[..]); + data +} + +fn key_val( + mut inp: &[u8], + len_key: usize, + len_val: usize, +) -> Vec<(Vec, Vec)> { + let mut result = Vec::new(); + loop { + let key = if inp.len() >= len_key { + inp[..len_key].to_vec() + } else { break }; + inp = &inp[len_key..]; + let val = if inp.len() >= len_val { + inp[..len_val].to_vec() + } else { break }; + inp = &inp[len_val..]; + result.push((key, val)); + } + result +} + +fn commit_drop_commit(b: &mut Bencher, input: &Vec) { + let key_vals = key_val(&input[..], 32, 32); + b.iter(move || { + let mut overlayed = OverlayedChanges::default(); + for i in key_vals.iter() { + BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.commit_prospective(); + for i in key_vals.iter() { + BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.discard_prospective(); + for i in key_vals.iter() { + BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.commit_prospective(); + }); +} + +fn commit_drop_commit_and_get(b: &mut Bencher, input: &Vec) { + let key_vals = key_val(&input[..], 32, 32); + b.iter(move || { + let mut overlayed = OverlayedChanges::default(); + for i in key_vals.iter() { + BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + } + for i in key_vals.iter() { + black_box(overlayed.storage(&i.0)); + } + overlayed.commit_prospective(); + for i in key_vals.iter() { + BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + } + for i in key_vals.iter() { + black_box(overlayed.storage(&i.0)); + } + overlayed.discard_prospective(); + for i in key_vals.iter() { + black_box(overlayed.storage(&i.0)); + } + for i in key_vals.iter() { + BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.commit_prospective(); + }); +} + + + + +fn bench_overlay_commit_drop_commit(c: &mut Criterion) { + let inp = get_content(12, CONTENT_KEY_SIZE * 100); + let inps = vec![inp]; + c.bench_function_over_inputs("commit_drop_commit", commit_drop_commit, inps); +} +fn bench_overlay_commit_drop_commit_get(c: &mut Criterion) { + let inp = get_content(12, CONTENT_KEY_SIZE * 100); + let inps = vec![inp]; + c.bench_function_over_inputs("commit_drop_commit_get", commit_drop_commit_and_get, inps); +} + + +criterion_group!(benches, + bench_overlay_commit_drop_commit, + bench_overlay_commit_drop_commit_get, +); +criterion_main!(benches); diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 7cff304675dce..995ff92402dff 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -53,6 +53,8 @@ pub use changes_trie::{ oldest_non_pruned_trie as oldest_non_pruned_changes_trie }; pub use overlayed_changes::OverlayedChanges; +#[cfg(feature = "bench")] +pub use overlayed_changes::BenchOverlay; pub use proving_backend::{ create_proof_check_backend, create_proof_check_backend_storage, Recorder as ProofRecorder, ProvingBackend, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 7d6d6081bd26b..9c532edcce001 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -291,6 +291,18 @@ impl OverlayedChanges { } } +#[cfg(feature = "bench")] +/// Expose private function of overlay for benching. +pub struct BenchOverlay<'a>(pub &'a mut OverlayedChanges); + + +#[cfg(feature = "bench")] +impl<'a> BenchOverlay<'a> { + pub fn bench_set_storage(&mut self, key: Vec, val: Option>) { + self.0.set_storage(key, val) + } +} + #[cfg(test)] impl From>> for OverlayedValue { fn from(value: Option>) -> OverlayedValue { From a218162f3d79a5c01911a2e963888c7f4449d37f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 9 Jul 2019 19:13:32 +0200 Subject: [PATCH 002/129] Init implementing. --- core/state-machine/src/overlayed_changes.rs | 338 ++++++++++++++------ 1 file changed, 235 insertions(+), 103 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 9c532edcce001..6ad9a95b849c1 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,16 +22,34 @@ use parity_codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; +#[derive(Debug, Clone)] +#[cfg_attr(test, derive(PartialEq))] +/// State of a transaction, the state are stored +/// and updated in a linear indexed way. +enum TransactionState { + /// Overlay is under change. + /// Is not necessarilly the last change. + /// Contains an index to commit position + Pending, + /// Information is committed, but can still be dropped. + Prospective, + /// Committed is information that cannot + /// be dropped. + Committed, + /// Transaction or prospective has been dropped. + Dropped, +} + /// The overlayed changes to state to be queried on top of the backend. /// /// A transaction shares all prospective changes within an inner overlay /// that can be cleared. #[derive(Debug, Default, Clone)] pub struct OverlayedChanges { - /// Changes that are not yet committed. - pub(crate) prospective: OverlayedChangeSet, - /// Committed changes. - pub(crate) committed: OverlayedChangeSet, +// /// History current position. +// pub(crate) current: usize, -> end of history vec + /// Changes with their history. + pub(crate) changes: OverlayedChangeSet, /// Changes trie configuration. None by default, but could be installed by the /// runtime if it supports change tries. pub(crate) changes_trie_config: Option, @@ -41,21 +59,34 @@ pub struct OverlayedChanges { #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedValue { - /// Current value. None if value has been deleted. + /// Values change historic with history state index. pub value: Option>, /// The set of extinsic indices where the values has been changed. - /// Is filled only if runtime has announced changes trie support. + /// Same history system as value (but over . pub extrinsics: Option>, } -/// Prospective or committed overlayed change set. +// TODO EMCH implement debug with latest vals only (at overlay changeset maybe). +#[derive(Debug, Default, Clone)] +#[cfg_attr(test, derive(PartialEq))] +struct History(pub Vec<(V, usize)>); + +/// Overlayed change set, keep history of value. #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { + /// Indexed changes history. + /// First index is initial state, + /// operation such as `init_transaction`, + /// `commit_transaction` or `drop_transaction` + /// will change this state. + pub history: Vec, + /// Size of change set, indexed at different history. + pub size: History, /// Top level storage changes. - pub top: HashMap, OverlayedValue>, + pub top: HashMap, History>, /// Child storage changes. - pub children: HashMap, (Option>, HashMap, Option>>)>, + pub children: HashMap, (HashMap, History>)>, } #[cfg(test)] @@ -68,23 +99,185 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { } } +impl History { + // TODO could have gc on get here we got some iteration that + // do not need to be repeated when tx dropped (history only + // updated on set). + fn get(&self, history: &[TransactionState]) -> Option<&V> { + // ix is never 0, + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= ix); + while ix > 0 { + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Pending + | TransactionState::Prospective + | TransactionState::Committed => + return Some(&self.0[ix].0), + TransactionState::Dropped => (), + } + ix -= 1; + } + None + } + fn into_committed(self, history: &[TransactionState]) -> Option { + // ix is never 0, + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= ix); + while ix > 0 { + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Committed => + return Some(self.0[ix].0), + TransactionState::Pending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + ix -= 1; + } + None + } + fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { + // ix is never 0, + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= ix); + while ix > 0 { + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Pending + | TransactionState::Prospective => { + return Some((&mut self.0[ix].0, self.0[ix].1)) + }, + | TransactionState::Committed => { + // No need for previous state + if ix > 0 { + self.0[0] = self.0[ix]; + self.0.truncate(1); + } + return Some((&mut self.0[0].0, self.0[0].1)) + }, + TransactionState::Dropped => { let _ = self.0.pop(); }, + } + ix -= 1; + } + None + } + + // should only be call after a get_mut check + fn push(&mut self, tx_ix: usize, val: V) { + self.0.push((val, tx_ix)) + } + + fn set(&mut self, history: &[TransactionState], val: V) { + match self.get_mut(history) { + Some((mut v, ix)) => { + if ix + 1 == history.len() { + *v = val; + return; + } + }, + None => (), + } + self.push(history.len() - 1, val); + } + +} + +impl History { + fn set_ext(&mut self, history: &[TransactionState], val: Option>, extrinsic_index: Option) { + if let Some((mut current, c_ix)) = self.get_mut(history) { + + if c_ix == history.len() - 1 { + current.value = val; + if let Some(extrinsic) = extrinsic_index { + current.extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic); + } + + } else { + let mut extrinsics = current.extrinsics.clone(); + extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic)); + self.push(history.len() - 1, OverlayedValue { + value: val, + extrinsics, + }); + } + } else { + let mut extrinsics: Option> = None; + extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic)); + + self.push(history.len() - 1, OverlayedValue { + value: val, + extrinsics, + }); + + } + } +} + impl OverlayedChangeSet { /// Whether the change set is empty. pub fn is_empty(&self) -> bool { - self.top.is_empty() && self.children.is_empty() + self.size.get(self.history.as_slice()).map(|v| *v == 0).unwrap_or(true) } /// Clear the change set. pub fn clear(&mut self) { + self.history.clear(); + self.size.0.clear(); self.top.clear(); self.children.clear(); } + + /// Discard prospective changes to state. + pub fn discard_prospective(&mut self) { + let mut i = self.history.len(); + while i > 0 { + i -= 1; + match self.history[i] { + TransactionState::Pending + | TransactionState::Prospective => self.history[i] = TransactionState::Dropped, + // can have dropped from transaction -> TODO make Dropped perspective variant? = earliest + // stop in some case + TransactionState::Dropped => (), + TransactionState::Committed => break, + } + } + self.history.push(TransactionState::Pending) + } + + /// Commit prospective changes to state. + pub fn commit_prospective(&mut self) { + debug_assert!(self.history.len() > 0); + // TODO EMCH can use offset and a GC state + self.history.last_mut().map(|mut v| *v = TransactionState::Committed); + } } impl OverlayedChanges { /// Whether the overlayed changes are empty. pub fn is_empty(&self) -> bool { - self.prospective.is_empty() && self.committed.is_empty() + self.changes.is_empty() } /// Sets the changes trie configuration. @@ -107,27 +300,25 @@ impl OverlayedChanges { /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. pub fn storage(&self, key: &[u8]) -> Option> { - self.prospective.top.get(key) - .or_else(|| self.committed.top.get(key)) - .map(|x| x.value.as_ref().map(AsRef::as_ref)) + if let Some(overlay_val) = self.changes.top.get(key) { + if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { + return Some(o_val.value.map(|v| v.as_slice())) + } + } + None } /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { - if let Some(map) = self.prospective.children.get(storage_key) { - if let Some(val) = map.1.get(key) { - return Some(val.as_ref().map(AsRef::as_ref)); - } - } - - if let Some(map) = self.committed.children.get(storage_key) { - if let Some(val) = map.1.get(key) { - return Some(val.as_ref().map(AsRef::as_ref)); - } - } - + if let Some(map) = self.changes.children.get(storage_key) { + if let Some(overlay_val) = map.get(key) { + if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { + return Some(o_val.value.map(|v| v.as_slice())) + } + } + } None } @@ -136,13 +327,8 @@ impl OverlayedChanges { /// `None` can be used to delete a value specified by the given key. pub(crate) fn set_storage(&mut self, key: Vec, val: Option>) { let extrinsic_index = self.extrinsic_index(); - let entry = self.prospective.top.entry(key).or_default(); - entry.value = val; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } + let entry = self.changes.top.entry(key).or_default(); + entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -150,14 +336,10 @@ impl OverlayedChanges { /// `None` can be used to delete a value specified by the given key. pub(crate) fn set_child_storage(&mut self, storage_key: Vec, key: Vec, val: Option>) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key).or_default(); - map_entry.1.insert(key, val); - - if let Some(extrinsic) = extrinsic_index { - map_entry.0.get_or_insert_with(Default::default) - .insert(extrinsic); - } - } + let map_entry = self.changes.children.entry(storage_key).or_default(); + let entry = map_entry.entry(key).or_default(); + entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); + } /// Clear child storage of given storage key. /// @@ -167,20 +349,9 @@ impl OverlayedChanges { /// [`discard_prospective`]: #method.discard_prospective pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key.to_vec()).or_default(); - - if let Some(extrinsic) = extrinsic_index { - map_entry.0.get_or_insert_with(Default::default) - .insert(extrinsic); - } - - map_entry.1.values_mut().for_each(|e| *e = None); + let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); - if let Some((_, committed_map)) = self.committed.children.get(storage_key) { - for (key, _) in committed_map.iter() { - map_entry.1.insert(key.clone(), None); - } - } + map_entry.values_mut().for_each(|e| e.set_ext(self.changes.history.as_slice(), None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -192,63 +363,21 @@ impl OverlayedChanges { pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); - // Iterate over all prospective and mark all keys that share - // the given prefix as removed (None). - for (key, entry) in self.prospective.top.iter_mut() { + for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - entry.value = None; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } - } - } - - // Then do the same with keys from commited changes. - // NOTE that we are making changes in the prospective change set. - for key in self.committed.top.keys() { - if key.starts_with(prefix) { - let entry = self.prospective.top.entry(key.clone()).or_default(); - entry.value = None; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } + entry.set_ext(self.changes.history.as_slice(), None, extrinsic_index); } } } /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { - self.prospective.clear(); + self.changes.discard_prospective(); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - if self.committed.is_empty() { - ::std::mem::swap(&mut self.prospective, &mut self.committed); - } else { - for (key, val) in self.prospective.top.drain() { - let entry = self.committed.top.entry(key).or_default(); - entry.value = val.value; - - if let Some(prospective_extrinsics) = val.extrinsics { - entry.extrinsics.get_or_insert_with(Default::default) - .extend(prospective_extrinsics); - } - } - for (storage_key, map) in self.prospective.children.drain() { - let entry = self.committed.children.entry(storage_key).or_default(); - entry.1.extend(map.1.iter().map(|(k, v)| (k.clone(), v.clone()))); - - if let Some(prospective_extrinsics) = map.0 { - entry.0.get_or_insert_with(Default::default) - .extend(prospective_extrinsics); - } - } - } + self.changes.commit_prospective(); } /// Consume `OverlayedChanges` and take committed set. @@ -259,9 +388,12 @@ impl OverlayedChanges { impl Iterator, Option>)>, impl Iterator, impl Iterator, Option>)>)>, ){ - assert!(self.prospective.is_empty()); - (self.committed.top.into_iter().map(|(k, v)| (k, v.value)), - self.committed.children.into_iter().map(|(sk, v)| (sk, v.1.into_iter()))) + ( + self.changes.top.into_iter() + .filter_map(|(k, v)| v.into_committed(self.changes.history.as_slice()).map(|v| (k, v.value))), + self.changes.children.into_iter().map(|(sk, v)| (sk, v.into_iter() + .filter_map(|(k, v)| v.into_committed(self.changes.history.as_slice()).map(|v| (k, v.value))))) + ) } /// Inserts storage entry responsible for current extrinsic index. From 4e8f6a722f13bf07f425edff785b42f8afe56820 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 11 Jul 2019 16:46:38 +0200 Subject: [PATCH 003/129] Initial implementation of prospective with unbound layer. --- core/state-machine/src/changes_trie/build.rs | 74 ++- core/state-machine/src/ext.rs | 63 +- core/state-machine/src/lib.rs | 40 +- core/state-machine/src/overlayed_changes.rs | 658 ++++++++++++------- core/state-machine/src/testing.rs | 15 +- 5 files changed, 539 insertions(+), 311 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 96d2278cc2a76..23f15df19715f 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -71,7 +71,7 @@ fn prepare_extrinsics_input( Number: BlockNumber, { let mut extrinsic_map = BTreeMap::, BTreeSet>::new(); - for (key, val) in changes.prospective.top.iter().chain(changes.committed.top.iter()) { + for (key, val) in changes.changes.top_iter_ext() { let extrinsics = match val.extrinsics { Some(ref extrinsics) => extrinsics, None => continue, @@ -146,7 +146,7 @@ mod test { use primitives::storage::well_known_keys::EXTRINSIC_INDEX; use crate::backend::InMemory; use crate::changes_trie::storage::InMemoryStorage; - use crate::overlayed_changes::OverlayedValue; + use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet, History, TransactionState}; use super::*; fn prepare_for_build() -> (InMemory, InMemoryStorage, OverlayedChanges) { @@ -193,31 +193,43 @@ mod test { (14, Vec::new()), (15, Vec::new()), ]); let changes = OverlayedChanges { - prospective: vec![ - (vec![100], OverlayedValue { - value: Some(vec![200]), - extrinsics: Some(vec![0, 2].into_iter().collect()) - }), - (vec![103], OverlayedValue { - value: None, - extrinsics: Some(vec![0, 1].into_iter().collect()) - }), - ].into_iter().collect(), - committed: vec![ - (EXTRINSIC_INDEX.to_vec(), OverlayedValue { - value: Some(3u32.encode()), - extrinsics: None, - }), - (vec![100], OverlayedValue { - value: Some(vec![202]), - extrinsics: Some(vec![3].into_iter().collect()) - }), - (vec![101], OverlayedValue { - value: Some(vec![203]), - extrinsics: Some(vec![1].into_iter().collect()) - }), - ].into_iter().collect(), changes_trie_config: Some(Configuration { digest_interval: 4, digest_levels: 2 }), + changes:OverlayedChangeSet { + history: vec![TransactionState::Committed, TransactionState::Pending], + state: 1, + size: History(vec![(3,0), (5, 0)]), + children: Default::default(), + top: vec![ + (EXTRINSIC_INDEX.to_vec(), History(vec![ + (OverlayedValue { + value: Some(3u32.encode()), + extrinsics: None, + }, 0), + ])), + (vec![100], History(vec![ + (OverlayedValue { + value: Some(vec![202]), + extrinsics: Some(vec![3].into_iter().collect()) + }, 0), + (OverlayedValue { + value: Some(vec![200]), + extrinsics: Some(vec![3, 0, 2].into_iter().collect()) + }, 1), + ])), + (vec![101], History(vec![ + (OverlayedValue { + value: Some(vec![203]), + extrinsics: Some(vec![1].into_iter().collect()) + }, 0), + ])), + (vec![103], History(vec![ + (OverlayedValue { + value: None, + extrinsics: Some(vec![0, 1].into_iter().collect()) + }, 1), + ])), + ].into_iter().collect(), + }, }; (backend, storage, changes) @@ -293,10 +305,12 @@ mod test { let (backend, storage, mut changes) = prepare_for_build(); // 110: missing from backend, set to None in overlay - changes.prospective.top.insert(vec![110], OverlayedValue { - value: None, - extrinsics: Some(vec![1].into_iter().collect()) - }); + changes.changes.top.insert(vec![110], History(vec![ + (OverlayedValue { + value: None, + extrinsics: Some(vec![1].into_iter().collect()), + }, 1), + ])); let config = changes.changes_trie_config.as_ref().unwrap(); let changes_trie_nodes = prepare_input( diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index d8619d1506d68..b51b26066efe5 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -154,8 +154,7 @@ where self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.committed.top.clone().into_iter().map(|(k, v)| (k, v.value))) - .chain(self.overlay.prospective.top.clone().into_iter().map(|(k, v)| (k, v.value))) + .chain(self.overlay.changes.top_iter().map(|(k, v)| (k.clone(), v.cloned()))) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -270,21 +269,15 @@ where } let child_storage_keys = - self.overlay.prospective.children.keys() - .chain(self.overlay.committed.children.keys()); + self.overlay.changes.children.keys(); let child_delta_iter = child_storage_keys.map(|storage_key| - (storage_key.clone(), self.overlay.committed.children.get(storage_key) - .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))) - .chain(self.overlay.prospective.children.get(storage_key) - .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone())))))); + (storage_key.clone(), self.overlay.changes.child_iter(storage_key) + .map(|(k, v)| (k.clone(), v.cloned())))); // compute and memoize - let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone())) - .chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))); + let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.clone(), v.cloned())); let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter); self.storage_transaction = Some((transaction, root)); @@ -302,12 +295,8 @@ where } else { let storage_key = storage_key.as_ref(); - let delta = self.overlay.committed.children.get(storage_key) - .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))) - .chain(self.overlay.prospective.children.get(storage_key) - .into_iter() - .flat_map(|map| map.1.clone().into_iter())); + let delta = self.overlay.changes.child_iter(storage_key) + .map(|(k, v)| (k.clone(), v.cloned())); let root = self.backend.child_storage_root(storage_key, delta).0; @@ -357,7 +346,7 @@ mod tests { use crate::backend::InMemory; use crate::changes_trie::{Configuration as ChangesTrieConfiguration, InMemoryStorage as InMemoryChangesTrieStorage}; - use crate::overlayed_changes::OverlayedValue; + use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet, History, TransactionState}; use super::*; type TestBackend = InMemory; @@ -365,22 +354,32 @@ mod tests { type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage, crate::NeverOffchainExt>; fn prepare_overlay_with_changes() -> OverlayedChanges { + OverlayedChanges { - prospective: vec![ - (EXTRINSIC_INDEX.to_vec(), OverlayedValue { - value: Some(3u32.encode()), - extrinsics: Some(vec![1].into_iter().collect()) - }), - (vec![1], OverlayedValue { - value: Some(vec![100].into_iter().collect()), - extrinsics: Some(vec![1].into_iter().collect()) - }), - ].into_iter().collect(), - committed: Default::default(), changes_trie_config: Some(ChangesTrieConfiguration { digest_interval: 0, digest_levels: 0, }), + changes:OverlayedChangeSet { + history: vec![TransactionState::Pending], + state: 0, + size: History(vec![(2,0)]), + children: Default::default(), + top: vec![ + (EXTRINSIC_INDEX.to_vec(), History(vec![ + (OverlayedValue { + value: Some(3u32.encode()), + extrinsics: Some(vec![1].into_iter().collect()) + }, 0), + ])), + (vec![1], History(vec![ + (OverlayedValue { + value: Some(vec![100]), + extrinsics: Some(vec![1].into_iter().collect()) + }, 0), + ])), + ].into_iter().collect(), + }, } } @@ -415,7 +414,9 @@ mod tests { #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { let mut overlay = prepare_overlay_with_changes(); - overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None; + let conf = overlay.remove_changes_trie_config().unwrap(); + overlay.set_storage(vec![1], None); + overlay.set_changes_trie_config(conf); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let backend = TestBackend::default(); let mut ext = TestExt::new(&mut overlay, &backend, Some(&storage), None); diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 995ff92402dff..a3d1957264b1d 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -565,7 +565,7 @@ impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where &mut self, compute_tx: bool, mut native_call: Option, - orig_prospective: OverlayedChangeSet, + orig_changes: OverlayedChangeSet, on_consensus_failure: Handler, ) -> (CallResult, Option<(B::Transaction, H::Out)>, Option>) where R: Decode + Encode + PartialEq, @@ -578,7 +578,7 @@ impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where let (result, was_native, storage_delta, changes_delta) = self.execute_aux(compute_tx, true, native_call.take()); if was_native { - self.overlay.prospective = orig_prospective.clone(); + self.overlay.changes = orig_changes.clone(); let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux(compute_tx, false, native_call); if (result.is_ok() && wasm_result.is_ok() @@ -597,7 +597,7 @@ impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where &mut self, compute_tx: bool, mut native_call: Option, - orig_prospective: OverlayedChangeSet, + orig_changes: OverlayedChangeSet, ) -> (CallResult, Option<(B::Transaction, H::Out)>, Option>) where R: Decode + Encode + PartialEq, NC: FnOnce() -> result::Result + UnwindSafe, @@ -607,12 +607,16 @@ impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where if !was_native || result.is_ok() { (result, storage_delta, changes_delta) } else { - self.overlay.prospective = orig_prospective.clone(); + self.overlay.changes = orig_changes.clone(); let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux(compute_tx, false, native_call); (wasm_result, wasm_storage_delta, wasm_changes_delta) } } + // TODO EMCH in all exec the prospective is replace by the full state (commited included) + // -> TODO bench over using a check point approach (expect ctommit prospective to not be call), + // or agains a gc checkpoint approach. + /// Execute a call using the given state backend, overlayed changes, and call executor. /// Produces a state-backend-specific "transaction" which can be used to apply the changes /// to the backing store, such as the disk. @@ -655,14 +659,14 @@ impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where init_overlay(self.overlay, false)?; let result = { - let orig_prospective = self.overlay.prospective.clone(); + let orig_changes = self.overlay.changes.clone(); let (result, storage_delta, changes_delta) = match manager { ExecutionManager::Both(on_consensus_failure) => { - self.execute_call_with_both_strategy(compute_tx, native_call.take(), orig_prospective, on_consensus_failure) + self.execute_call_with_both_strategy(compute_tx, native_call.take(), orig_changes, on_consensus_failure) }, ExecutionManager::NativeElseWasm => { - self.execute_call_with_native_else_wasm_strategy(compute_tx, native_call.take(), orig_prospective) + self.execute_call_with_native_else_wasm_strategy(compute_tx, native_call.take(), orig_changes) }, ExecutionManager::AlwaysWasm => { let (result, _, storage_delta, changes_delta) = self.execute_aux(compute_tx, false, native_call); @@ -1121,17 +1125,14 @@ mod tests { ]; let mut state = InMemory::::from(initial); let backend = state.as_trie_backend().unwrap(); - let mut overlay = OverlayedChanges { - committed: map![ - b"aba".to_vec() => OverlayedValue::from(Some(b"1312".to_vec())), - b"bab".to_vec() => OverlayedValue::from(Some(b"228".to_vec())) - ], - prospective: map![ - b"abd".to_vec() => OverlayedValue::from(Some(b"69".to_vec())), - b"bbd".to_vec() => OverlayedValue::from(Some(b"42".to_vec())) + let mut overlay = OverlayedChanges::new_from_top(vec![ + (b"aba".to_vec(), Some(b"1312".to_vec())), + (b"bab".to_vec(), Some(b"228".to_vec())), ], - ..Default::default() - }; + vec![ + (b"abd".to_vec(), Some(b"69".to_vec())), + (b"bbd".to_vec(), Some(b"42".to_vec())), + ]); { let changes_trie_storage = InMemoryChangesTrieStorage::::new(); @@ -1140,8 +1141,11 @@ mod tests { } overlay.commit_prospective(); + overlay.discard_prospective(); + let values: HashMap<_, _> = overlay.changes.top_iter() + .map(|(k, v)| (k.clone(), v.cloned())).collect(); assert_eq!( - overlay.committed, + values, map![ b"abc".to_vec() => None.into(), b"abb".to_vec() => None.into(), diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 6ad9a95b849c1..2735f8cffd44e 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -26,18 +26,18 @@ use primitives::storage::well_known_keys::EXTRINSIC_INDEX; #[cfg_attr(test, derive(PartialEq))] /// State of a transaction, the state are stored /// and updated in a linear indexed way. -enum TransactionState { - /// Overlay is under change. - /// Is not necessarilly the last change. - /// Contains an index to commit position - Pending, - /// Information is committed, but can still be dropped. - Prospective, - /// Committed is information that cannot - /// be dropped. - Committed, - /// Transaction or prospective has been dropped. - Dropped, +pub(crate) enum TransactionState { + /// Overlay is under change. + /// Is not necessarilly the last change. + /// Contains an index to commit position + Pending, + /// Information is committed, but can still be dropped. + Prospective, + /// Committed is information that cannot + /// be dropped. + Committed, + /// Transaction or prospective has been dropped. + Dropped, } /// The overlayed changes to state to be queried on top of the backend. @@ -46,7 +46,7 @@ enum TransactionState { /// that can be cleared. #[derive(Debug, Default, Clone)] pub struct OverlayedChanges { -// /// History current position. +// /// History current position. // pub(crate) current: usize, -> end of history vec /// Changes with their history. pub(crate) changes: OverlayedChangeSet, @@ -67,211 +67,378 @@ pub struct OverlayedValue { } // TODO EMCH implement debug with latest vals only (at overlay changeset maybe). +// this is pub crate for direct building in test (same with fields) -> TODO +// make fields private and have a build method and make history private?. #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] -struct History(pub Vec<(V, usize)>); +pub(crate) struct History(pub Vec<(V, usize)>); /// Overlayed change set, keep history of value. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { /// Indexed changes history. - /// First index is initial state, - /// operation such as `init_transaction`, - /// `commit_transaction` or `drop_transaction` - /// will change this state. - pub history: Vec, - /// Size of change set, indexed at different history. - pub size: History, + /// First index is initial state, + /// operation such as `init_transaction`, + /// `commit_transaction` or `drop_transaction` + /// will change this state. + pub(crate) history: Vec, + /// Current state in history. + pub(crate) state: usize, + /// Size of change set, indexed at different history. + /// TODO EMCH probably not needed overhead. It is only to skip processing + /// when empty: remove and change size method to is empty with no guaranties (just + /// touched on first insert aka top and bottom empty as before)? + pub(crate) size: History, /// Top level storage changes. - pub top: HashMap, History>, + pub(crate) top: HashMap, History>, /// Child storage changes. - pub children: HashMap, (HashMap, History>)>, + pub(crate) children: HashMap, (HashMap, History>)>, +} + +impl Default for OverlayedChangeSet { + fn default() -> Self { + OverlayedChangeSet { + history: vec![TransactionState::Pending], + state: 0, + size: History(vec![(0,0)]), + top: Default::default(), + children: Default::default(), + } + } } #[cfg(test)] impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { - Self { - top: iter.into_iter().collect(), - children: Default::default(), - } + + let mut result = OverlayedChangeSet::default(); + result.top = iter.into_iter().map(|(k, v)| (k, History(vec![(v, 0)]))).collect(); + result.size = History(vec![(result.top.len(), 0)]); + result } } impl History { - // TODO could have gc on get here we got some iteration that - // do not need to be repeated when tx dropped (history only - // updated on set). - fn get(&self, history: &[TransactionState]) -> Option<&V> { - // ix is never 0, - let mut ix = self.0.len(); - if ix == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= ix); - while ix > 0 { - let hix = self.0[ix].1; - match history[hix] { - TransactionState::Pending - | TransactionState::Prospective - | TransactionState::Committed => - return Some(&self.0[ix].0), - TransactionState::Dropped => (), - } - ix -= 1; - } - None - } - fn into_committed(self, history: &[TransactionState]) -> Option { - // ix is never 0, - let mut ix = self.0.len(); - if ix == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= ix); - while ix > 0 { - let hix = self.0[ix].1; - match history[hix] { - TransactionState::Committed => - return Some(self.0[ix].0), - TransactionState::Pending - | TransactionState::Prospective - | TransactionState::Dropped => (), - } - ix -= 1; - } - None - } - fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { - // ix is never 0, - let mut ix = self.0.len(); - if ix == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= ix); - while ix > 0 { - let hix = self.0[ix].1; - match history[hix] { - TransactionState::Pending - | TransactionState::Prospective => { - return Some((&mut self.0[ix].0, self.0[ix].1)) - }, - | TransactionState::Committed => { - // No need for previous state - if ix > 0 { - self.0[0] = self.0[ix]; - self.0.truncate(1); - } - return Some((&mut self.0[0].0, self.0[0].1)) - }, - TransactionState::Dropped => { let _ = self.0.pop(); }, - } - ix -= 1; - } - None - } + // TODO could have gc on get here we got some iteration that + // do not need to be repeated when tx dropped (history only + // updated on set). + fn get(&self, history: &[TransactionState]) -> Option<&V> { + // ix is never 0, + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= ix); + while ix > 0 { + ix -= 1; + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Pending + | TransactionState::Prospective + | TransactionState::Committed => + return Some(&self.0[ix].0), + TransactionState::Dropped => (), + } + } + None + } - // should only be call after a get_mut check - fn push(&mut self, tx_ix: usize, val: V) { - self.0.push((val, tx_ix)) - } + #[cfg(test)] + fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { + // ix is never 0, + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + debug_assert!(history.len() >= ix); + while ix > 0 { + ix -= 1; + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Pending + | TransactionState::Prospective => + return Some(&self.0[ix].0), + TransactionState::Committed + | TransactionState::Dropped => (), + } + } + None + } - fn set(&mut self, history: &[TransactionState], val: V) { - match self.get_mut(history) { - Some((mut v, ix)) => { - if ix + 1 == history.len() { - *v = val; - return; - } - }, - None => (), - } - self.push(history.len() - 1, val); - } + #[cfg(test)] + fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { + // ix is never 0, + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + debug_assert!(history.len() >= ix); + while ix > 0 { + ix -= 1; + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Committed => + return Some(&self.0[ix].0), + TransactionState::Pending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + + + fn into_committed(mut self, history: &[TransactionState]) -> Option { + // ix is never 0, + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= ix); + while ix > 0 { + ix -= 1; + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Committed => { + self.0.truncate(ix + 1); + return self.0.pop().map(|v| v.0); + }, + TransactionState::Pending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { + + let mut ix = self.0.len(); + if ix == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= ix); + while ix > 0 { + ix -= 1; + let hix = self.0[ix].1; + match history[hix] { + TransactionState::Pending + | TransactionState::Prospective => { + let tx = self.0[ix].1; + return Some((&mut self.0[ix].0, tx)) + }, + | TransactionState::Committed => { + // No need for previous state + if ix > 0 { + self.0.truncate(ix + 1); + let tv = self.0.pop().expect("just checked"); + self.0[0] = tv; + self.0.truncate(1); + } + let tx = self.0[ix].1; + return Some((&mut self.0[0].0, tx)) + }, + TransactionState::Dropped => { let _ = self.0.pop(); }, + } + } + None + } + + // should only be call after a get_mut check + fn push(&mut self, val:V, tx_ix: usize) { + self.0.push((val, tx_ix)) + } + + fn set(&mut self, history: &[TransactionState], state: usize, val: V) { + // TODO EMCH : this sis not optimal : can end get_mut as soon as ix < state + // needs a variant for get_mut. + match self.get_mut(history) { + Some((v, ix)) => { + if ix == state { + *v = val; + return; + } + }, + None => (), + } + self.push(val, history.len() - 1); + } } impl History { - fn set_ext(&mut self, history: &[TransactionState], val: Option>, extrinsic_index: Option) { - if let Some((mut current, c_ix)) = self.get_mut(history) { - - if c_ix == history.len() - 1 { - current.value = val; - if let Some(extrinsic) = extrinsic_index { - current.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } - - } else { - let mut extrinsics = current.extrinsics.clone(); - extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic)); - self.push(history.len() - 1, OverlayedValue { - value: val, - extrinsics, - }); - } - } else { - let mut extrinsics: Option> = None; - extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic)); - - self.push(history.len() - 1, OverlayedValue { - value: val, - extrinsics, - }); + fn set_ext(&mut self, history: &[TransactionState], state: usize, val: Option>, extrinsic_index: Option) { + if let Some((mut current, c_ix)) = self.get_mut(history) { + + if c_ix == state { + current.value = val; + if let Some(extrinsic) = extrinsic_index { + current.extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic); + } + + } else { + let mut extrinsics = current.extrinsics.clone(); + extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic)); + self.push(OverlayedValue { + value: val, + extrinsics, + }, state); + } + } else { + let mut extrinsics: Option> = None; + extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic)); + + self.push(OverlayedValue { + value: val, + extrinsics, + }, state); - } - } + } + } } impl OverlayedChangeSet { /// Whether the change set is empty. pub fn is_empty(&self) -> bool { - self.size.get(self.history.as_slice()).map(|v| *v == 0).unwrap_or(true) + self.size.get(self.history.as_slice()).map(|v| *v == 0).unwrap_or(true) } /// Clear the change set. pub fn clear(&mut self) { - self.history.clear(); - self.size.0.clear(); + self.history = vec![TransactionState::Pending]; + self.state = 0; + self.size.0.clear(); + self.size.push(0,0); self.top.clear(); self.children.clear(); } + fn increase_size(&mut self, nb: usize) { + let init = if let Some((s, c_ix)) = self.size.get_mut(self.history.as_slice()) { + if self.state == c_ix { + *s += nb; + return; + } else { + *s + } + } else { 0 }; + self.size.push(init + nb, self.state); + } + + fn decrease_size(&mut self, nb: usize) { + let init = if let Some((s, c_ix)) = self.size.get_mut(self.history.as_slice()) { + if self.state == c_ix { + *s -= nb; + return; + } else { + *s + } + } else { + debug_assert!(false, "decrease size on non initialize size is a bug"); + nb + }; + self.size.push(init - nb, self.state); + } + /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { - let mut i = self.history.len(); - while i > 0 { - i -= 1; - match self.history[i] { - TransactionState::Pending - | TransactionState::Prospective => self.history[i] = TransactionState::Dropped, - // can have dropped from transaction -> TODO make Dropped perspective variant? = earliest - // stop in some case - TransactionState::Dropped => (), - TransactionState::Committed => break, - } - } - self.history.push(TransactionState::Pending) - } + let mut i = self.state + 1; + while i > 0 { + i -= 1; + match self.history[i] { + TransactionState::Pending + | TransactionState::Prospective => self.history[i] = TransactionState::Dropped, + // can have dropped from transaction -> TODO make Dropped perspective variant? = earliest + // stop in some case + TransactionState::Dropped => (), + TransactionState::Committed => break, + } + } + self.history.push(TransactionState::Pending); + self.state = self.history.len() - 1; + } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - debug_assert!(self.history.len() > 0); - // TODO EMCH can use offset and a GC state - self.history.last_mut().map(|mut v| *v = TransactionState::Committed); + debug_assert!(self.history.len() > 0); + // TODO EMCH can use offset and a GC state + self.history[self.state] = TransactionState::Committed; + self.history.push(TransactionState::Pending); + self.state = self.history.len() - 1; + } + + /// Iterator over current state of the overlay. + pub fn top_iter_ext(&self) -> impl Iterator, &OverlayedValue)> { + self.top.iter() + .filter_map(move |(k, v)| + v.get(self.history.as_slice()).map(|v| (k, v))) + } + + + /// Iterator over current state of the overlay. + pub fn top_iter(&self) -> impl Iterator, Option<&Vec>)> { + self.top_iter_ext().map(|(k, v)| (k, v.value.as_ref())) + } + + /// Iterator over current state of the overlay. + pub fn child_iter_ext( + &self, + storage_key: &[u8], + ) -> impl Iterator, &OverlayedValue)> { + self.children.get(storage_key) + .into_iter() + .flat_map(move |child| child.iter() + .filter_map(move |(k, v)| + v.get(self.history.as_slice()).map(|v| (k, v))) + ) + } + + /// Iterator over current state of the overlay. + pub fn child_iter(&self, storage_key: &[u8]) -> impl Iterator, Option<&Vec>)> { + self.child_iter_ext(storage_key).map(|(k, v)| (k, v.value.as_ref())) + } + + /// Test only method to access current prospective changes. + /// It is here to keep old test compatibility and should be + /// avoid for new tests. + #[cfg(test)] + pub(crate) fn top_prospective(&self) -> HashMap, OverlayedValue> { + let mut result = HashMap::new(); + for (k, v) in self.top.iter() { + if let Some(v) = v.get_prospective(self.history.as_slice()) { + result.insert(k.clone(), v.clone()); + } + } + result } + /// Test only method to access current commited changes. + /// It is here to keep old test compatibility and should be + /// avoid for new tests. + #[cfg(test)] + pub(crate) fn top_committed(&self) -> HashMap, OverlayedValue> { + let mut result = HashMap::new(); + for (k, v) in self.top.iter() { + if let Some(v) = v.get_committed(self.history.as_slice()) { + result.insert(k.clone(), v.clone()); + } + } + result + } + } impl OverlayedChanges { @@ -295,17 +462,23 @@ impl OverlayedChanges { self.changes_trie_config = Some(config); true } + /// To make some non followed change + #[cfg(test)] + pub(crate) fn remove_changes_trie_config(&mut self) -> Option { + self.changes_trie_config.take() + } + /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. pub fn storage(&self, key: &[u8]) -> Option> { - if let Some(overlay_val) = self.changes.top.get(key) { - if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { - return Some(o_val.value.map(|v| v.as_slice())) - } - } - None + if let Some(overlay_val) = self.changes.top.get(key) { + if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { + return Some(o_val.value.as_ref().map(|v| v.as_slice())) + } + } + None } /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered @@ -313,12 +486,12 @@ impl OverlayedChanges { /// value has been set. pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.changes.children.get(storage_key) { - if let Some(overlay_val) = map.get(key) { - if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { - return Some(o_val.value.map(|v| v.as_slice())) - } - } - } + if let Some(overlay_val) = map.get(key) { + if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { + return Some(o_val.value.as_ref().map(|v| v.as_slice())) + } + } + } None } @@ -328,7 +501,8 @@ impl OverlayedChanges { pub(crate) fn set_storage(&mut self, key: Vec, val: Option>) { let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); + entry.set_ext(self.changes.history.as_slice(), self.changes.state, val, extrinsic_index); + self.changes.increase_size(1); } /// Inserts the given key-value pair into the prospective child change set. @@ -338,8 +512,9 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); - entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); - } + entry.set_ext(self.changes.history.as_slice(), self.changes.state, val, extrinsic_index); + self.changes.increase_size(1); + } /// Clear child storage of given storage key. /// @@ -349,9 +524,11 @@ impl OverlayedChanges { /// [`discard_prospective`]: #method.discard_prospective pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) { let extrinsic_index = self.extrinsic_index(); + let history = self.changes.history.as_slice(); + let state = self.changes.state; let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); - map_entry.values_mut().for_each(|e| e.set_ext(self.changes.history.as_slice(), None, extrinsic_index)); + map_entry.values_mut().for_each(|e| e.set_ext(history, state, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -365,19 +542,19 @@ impl OverlayedChanges { for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - entry.set_ext(self.changes.history.as_slice(), None, extrinsic_index); + entry.set_ext(self.changes.history.as_slice(), self.changes.state, None, extrinsic_index); } } } /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { - self.changes.discard_prospective(); + self.changes.discard_prospective(); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - self.changes.commit_prospective(); + self.changes.commit_prospective(); } /// Consume `OverlayedChanges` and take committed set. @@ -388,24 +565,66 @@ impl OverlayedChanges { impl Iterator, Option>)>, impl Iterator, impl Iterator, Option>)>)>, ){ - ( - self.changes.top.into_iter() - .filter_map(|(k, v)| v.into_committed(self.changes.history.as_slice()).map(|v| (k, v.value))), - self.changes.children.into_iter().map(|(sk, v)| (sk, v.into_iter() - .filter_map(|(k, v)| v.into_committed(self.changes.history.as_slice()).map(|v| (k, v.value))))) - ) + let top = self.changes.top; + let children = self.changes.children; + let history = self.changes.history.clone(); + let history2 = self.changes.history; + ( + top.into_iter() + .filter_map(move |(k, v)| v.into_committed(history.as_slice()).map(|v| (k, v.value))), + children.into_iter().map(move |(sk, v)| { + let historys = history2.clone(); + (sk, v.into_iter() + .filter_map(move |(k, v)| v.into_committed(historys.as_slice()).map(|v| (k, v.value)))) + }) + ) } /// Inserts storage entry responsible for current extrinsic index. #[cfg(test)] pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) { use parity_codec::Encode; - self.prospective.top.insert(EXTRINSIC_INDEX.to_vec(), OverlayedValue { - value: Some(extrinsic_index.encode()), - extrinsics: None, - }); + + let value = extrinsic_index.encode(); + let entry = self.changes.top.entry(EXTRINSIC_INDEX.to_vec()).or_default(); + entry.set_ext(self.changes.history.as_slice(), self.changes.state, Some(value), None); } + /// Test only method to build from committed info and prospective. + /// Create an history of two states. + #[cfg(test)] + pub(crate) fn new_from_top( + committed: Vec<(Vec, Option>)>, + prospective: Vec<(Vec, Option>)>, + ) -> Self { + let mut changes = OverlayedChangeSet::default(); + changes.top = committed.into_iter().map(|(k, v)| + (k, History(vec![(OverlayedValue { value: v, extrinsics: None }, 0)]))).collect(); + changes.size = History(vec![(changes.top.len(), 0)]); + changes.commit_prospective(); + let mut result = OverlayedChanges { changes, changes_trie_config: None }; + prospective.into_iter().for_each(|(k, v)| result.set_storage(k, v)); + result + } + /// Test only method to build from committed info and prospective. + /// Create an history of two states. + /// TODO EMCH replace prev fn with this one. + #[cfg(test)] + pub(crate) fn new_from_top_changes( + committed: Vec<(Vec, Option>)>, + prospective: Vec<(Vec, Option>)>, + changes_trie_config: Option, + ) -> Self { + let mut changes = OverlayedChangeSet::default(); + let mut result = OverlayedChanges { changes, changes_trie_config }; + committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); + result.changes.commit_prospective(); + prospective.into_iter().for_each(|(k, v)| result.set_storage(k, v)); + result + } + + + /// Returns current extrinsic index to use in changes trie construction. /// None is returned if it is not set or changes trie config is not set. /// Persistent value (from the backend) can be ignored because runtime must @@ -453,10 +672,9 @@ mod tests { use crate::Externalities; use super::*; - fn strip_extrinsic_index(map: &HashMap, OverlayedValue>) -> HashMap, OverlayedValue> { - let mut clone = map.clone(); - clone.remove(&EXTRINSIC_INDEX.to_vec()); - clone + fn strip_extrinsic_index(mut map: HashMap, OverlayedValue>) -> HashMap, OverlayedValue> { + map.remove(&EXTRINSIC_INDEX.to_vec()); + map } #[test] @@ -496,18 +714,16 @@ mod tests { (b"doug".to_vec(), b"notadog".to_vec()), ].into_iter().collect(); let backend = InMemory::::from(initial); - let mut overlay = OverlayedChanges { - committed: vec![ + let mut overlay = OverlayedChanges::new_from_top( + vec![ (b"dog".to_vec(), Some(b"puppy".to_vec()).into()), (b"dogglesworth".to_vec(), Some(b"catYYY".to_vec()).into()), (b"doug".to_vec(), Some(vec![]).into()), ].into_iter().collect(), - prospective: vec![ + vec![ (b"dogglesworth".to_vec(), Some(b"cat".to_vec()).into()), (b"doug".to_vec(), None.into()), - ].into_iter().collect(), - ..Default::default() - }; + ].into_iter().collect()); let changes_trie_storage = InMemoryChangesTrieStorage::::new(); let mut ext = Ext::new( @@ -543,7 +759,7 @@ mod tests { digest_interval: 4, digest_levels: 1, }), true); assert_eq!( - strip_extrinsic_index(&overlay.prospective.top), + strip_extrinsic_index(overlay.changes.top_prospective()), vec![ (vec![1], OverlayedValue { value: Some(vec![2]), extrinsics: Some(vec![0].into_iter().collect()) }), ].into_iter().collect(), @@ -579,7 +795,7 @@ mod tests { overlay.set_extrinsic_index(2); overlay.set_storage(vec![1], Some(vec![6])); - assert_eq!(strip_extrinsic_index(&overlay.prospective.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_prospective()), vec![ (vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }), (vec![3], OverlayedValue { value: Some(vec![4]), extrinsics: Some(vec![1].into_iter().collect()) }), @@ -594,29 +810,29 @@ mod tests { overlay.set_extrinsic_index(4); overlay.set_storage(vec![1], Some(vec![8])); - assert_eq!(strip_extrinsic_index(&overlay.committed.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_committed()), vec![ (vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }), (vec![3], OverlayedValue { value: Some(vec![4]), extrinsics: Some(vec![1].into_iter().collect()) }), (vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }), ].into_iter().collect()); - assert_eq!(strip_extrinsic_index(&overlay.prospective.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_prospective()), vec![ - (vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![4].into_iter().collect()) }), - (vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![3].into_iter().collect()) }), + (vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }), + (vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![1, 3].into_iter().collect()) }), ].into_iter().collect()); overlay.commit_prospective(); - assert_eq!(strip_extrinsic_index(&overlay.committed.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_committed()), vec![ (vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }), (vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![1, 3].into_iter().collect()) }), (vec![100], OverlayedValue { value: Some(vec![101]), extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }), ].into_iter().collect()); - assert_eq!(overlay.prospective, + assert_eq!(overlay.changes.top_prospective(), Default::default()); } } diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 759ffb6b961aa..46982484be40d 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -76,8 +76,7 @@ impl TestExternalities { pub fn iter_pairs_in_order(&self) -> impl Iterator, Vec)> { self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.committed.top.clone().into_iter().map(|(k, v)| (k, v.value))) - .chain(self.overlay.prospective.top.clone().into_iter().map(|(k, v)| (k, v.value))) + .chain(self.overlay.changes.top_iter().map(|(k, v)| (k.clone(), v.cloned()))) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -202,8 +201,7 @@ impl Externalities for TestExternalities fn storage_root(&mut self) -> H::Out { // compute and memoize - let delta = self.overlay.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone())) - .chain(self.overlay.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))); + let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.clone(), v.cloned())); self.backend.storage_root(delta).0 } @@ -212,13 +210,8 @@ impl Externalities for TestExternalities let storage_key = storage_key.as_ref(); let (root, _, _) = { - let delta = self.overlay.committed.children.get(storage_key) - .into_iter() - .flat_map(|map| map.1.iter().map(|(k, v)| (k.clone(), v.clone()))) - .chain(self.overlay.prospective.children.get(storage_key) - .into_iter() - .flat_map(|map| map.1.clone().into_iter())); - + let delta = self.overlay.changes.child_iter(storage_key) + .map(|(k, v)| (k.clone(), v.cloned())); self.backend.child_storage_root(storage_key, delta) }; self.overlay.set_storage(storage_key.into(), Some(root.clone())); From 917e26bc951e64267ffaf3196845bc47a661eded Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 11 Jul 2019 17:04:37 +0200 Subject: [PATCH 004/129] Remove size counter --- core/state-machine/src/changes_trie/build.rs | 1 - core/state-machine/src/ext.rs | 1 - core/state-machine/src/overlayed_changes.rs | 41 +------------------- 3 files changed, 1 insertion(+), 42 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 23f15df19715f..5904f8c461a93 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -197,7 +197,6 @@ mod test { changes:OverlayedChangeSet { history: vec![TransactionState::Committed, TransactionState::Pending], state: 1, - size: History(vec![(3,0), (5, 0)]), children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), History(vec![ diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index b51b26066efe5..45c1721683f50 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -363,7 +363,6 @@ mod tests { changes:OverlayedChangeSet { history: vec![TransactionState::Pending], state: 0, - size: History(vec![(2,0)]), children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), History(vec![ diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 2735f8cffd44e..f407670801f08 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -85,11 +85,6 @@ pub struct OverlayedChangeSet { pub(crate) history: Vec, /// Current state in history. pub(crate) state: usize, - /// Size of change set, indexed at different history. - /// TODO EMCH probably not needed overhead. It is only to skip processing - /// when empty: remove and change size method to is empty with no guaranties (just - /// touched on first insert aka top and bottom empty as before)? - pub(crate) size: History, /// Top level storage changes. pub(crate) top: HashMap, History>, /// Child storage changes. @@ -101,7 +96,6 @@ impl Default for OverlayedChangeSet { OverlayedChangeSet { history: vec![TransactionState::Pending], state: 0, - size: History(vec![(0,0)]), top: Default::default(), children: Default::default(), } @@ -114,7 +108,6 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, v)| (k, History(vec![(v, 0)]))).collect(); - result.size = History(vec![(result.top.len(), 0)]); result } } @@ -314,46 +307,17 @@ impl History { impl OverlayedChangeSet { /// Whether the change set is empty. pub fn is_empty(&self) -> bool { - self.size.get(self.history.as_slice()).map(|v| *v == 0).unwrap_or(true) + self.top.is_empty() && self.children.is_empty() } /// Clear the change set. pub fn clear(&mut self) { self.history = vec![TransactionState::Pending]; self.state = 0; - self.size.0.clear(); - self.size.push(0,0); self.top.clear(); self.children.clear(); } - fn increase_size(&mut self, nb: usize) { - let init = if let Some((s, c_ix)) = self.size.get_mut(self.history.as_slice()) { - if self.state == c_ix { - *s += nb; - return; - } else { - *s - } - } else { 0 }; - self.size.push(init + nb, self.state); - } - - fn decrease_size(&mut self, nb: usize) { - let init = if let Some((s, c_ix)) = self.size.get_mut(self.history.as_slice()) { - if self.state == c_ix { - *s -= nb; - return; - } else { - *s - } - } else { - debug_assert!(false, "decrease size on non initialize size is a bug"); - nb - }; - self.size.push(init - nb, self.state); - } - /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { let mut i = self.state + 1; @@ -502,7 +466,6 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); entry.set_ext(self.changes.history.as_slice(), self.changes.state, val, extrinsic_index); - self.changes.increase_size(1); } /// Inserts the given key-value pair into the prospective child change set. @@ -513,7 +476,6 @@ impl OverlayedChanges { let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); entry.set_ext(self.changes.history.as_slice(), self.changes.state, val, extrinsic_index); - self.changes.increase_size(1); } /// Clear child storage of given storage key. @@ -600,7 +562,6 @@ impl OverlayedChanges { let mut changes = OverlayedChangeSet::default(); changes.top = committed.into_iter().map(|(k, v)| (k, History(vec![(OverlayedValue { value: v, extrinsics: None }, 0)]))).collect(); - changes.size = History(vec![(changes.top.len(), 0)]); changes.commit_prospective(); let mut result = OverlayedChanges { changes, changes_trie_config: None }; prospective.into_iter().for_each(|(k, v)| result.set_storage(k, v)); From 8b93ab7e968e9a62f9c334fe48a33a4997c593a3 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 11 Jul 2019 17:41:48 +0200 Subject: [PATCH 005/129] fix ix. --- core/state-machine/src/overlayed_changes.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index f407670801f08..f4fcc4a84db45 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -239,7 +239,7 @@ impl History { self.0[0] = tv; self.0.truncate(1); } - let tx = self.0[ix].1; + let tx = self.0[0].1; return Some((&mut self.0[0].0, tx)) }, TransactionState::Dropped => { let _ = self.0.pop(); }, @@ -254,8 +254,8 @@ impl History { } fn set(&mut self, history: &[TransactionState], state: usize, val: V) { - // TODO EMCH : this sis not optimal : can end get_mut as soon as ix < state - // needs a variant for get_mut. + // TODO EMCH : this is not optimal : can end get_mut as soon as ix < state + // needs a variant for get_mut. match self.get_mut(history) { Some((v, ix)) => { if ix == state { From ae8e880e76b55c651335fef32a4b740645b48c72 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 12 Jul 2019 09:38:59 +0200 Subject: [PATCH 006/129] code for transactional layers: TODO testing!! --- core/state-machine/src/overlayed_changes.rs | 53 ++++++++++++++++++--- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index f4fcc4a84db45..5102ff8ac3f8f 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -83,7 +83,7 @@ pub struct OverlayedChangeSet { /// `commit_transaction` or `drop_transaction` /// will change this state. pub(crate) history: Vec, - /// Current state in history. + /// Current state in history. pub(crate) state: usize, /// Top level storage changes. pub(crate) top: HashMap, History>, @@ -106,9 +106,9 @@ impl Default for OverlayedChangeSet { impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { - let mut result = OverlayedChangeSet::default(); + let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, v)| (k, History(vec![(v, 0)]))).collect(); - result + result } } @@ -254,8 +254,8 @@ impl History { } fn set(&mut self, history: &[TransactionState], state: usize, val: V) { - // TODO EMCH : this is not optimal : can end get_mut as soon as ix < state - // needs a variant for get_mut. + // TODO EMCH : this is not optimal : can end get_mut as soon as ix < state + // needs a variant for get_mut. match self.get_mut(history) { Some((v, ix)) => { if ix == state { @@ -334,7 +334,7 @@ impl OverlayedChangeSet { } self.history.push(TransactionState::Pending); self.state = self.history.len() - 1; - } + } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { @@ -345,6 +345,45 @@ impl OverlayedChangeSet { self.state = self.history.len() - 1; } + /// Create a new transactional layre + pub fn start_transaction(&mut self) { + self.history.push(TransactionState::Pending); + self.state = self.history.len() - 1; + } + + + /// Discard a transactional layer. + /// A transaction is always running (history always end with pending). + pub fn discard_transaction(&mut self) { + debug_assert!(self.history.len() > self.state); + self.history[self.state] = TransactionState::Dropped; + let mut i = self.state; + // revert state to previuos pending (or create new pending) + while i > 0 { + i -= 1; + match self.history[i] { + TransactionState::Pending => { + self.state = i; + return; + }, + TransactionState::Dropped => (), + TransactionState::Prospective + | TransactionState::Committed => break, + } + } + self.history.push(TransactionState::Pending); + self.state = self.history.len() - 1; + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + debug_assert!(self.history.len() > self.state); + self.history[self.state] = TransactionState::Prospective; + self.history.push(TransactionState::Pending); + self.state = self.history.len() - 1; + } + + /// Iterator over current state of the overlay. pub fn top_iter_ext(&self) -> impl Iterator, &OverlayedValue)> { self.top.iter() @@ -559,7 +598,7 @@ impl OverlayedChanges { committed: Vec<(Vec, Option>)>, prospective: Vec<(Vec, Option>)>, ) -> Self { - let mut changes = OverlayedChangeSet::default(); + let mut changes = OverlayedChangeSet::default(); changes.top = committed.into_iter().map(|(k, v)| (k, History(vec![(OverlayedValue { value: v, extrinsics: None }, 0)]))).collect(); changes.commit_prospective(); From c716f958a8c9376f98ee0477cd356c0fff0aadaa Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 30 Jul 2019 10:19:49 +0200 Subject: [PATCH 007/129] Regain perfs (-15%) by using a couple of static memory for initial values. --- core/state-machine/src/changes_trie/build.rs | 10 +- core/state-machine/src/ext.rs | 4 +- core/state-machine/src/overlayed_changes.rs | 150 ++++++++++++++----- 3 files changed, 121 insertions(+), 43 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 5904f8c461a93..3639a561f4a3d 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -199,13 +199,13 @@ mod test { state: 1, children: Default::default(), top: vec![ - (EXTRINSIC_INDEX.to_vec(), History(vec![ + (EXTRINSIC_INDEX.to_vec(), History::from_vec(vec![ (OverlayedValue { value: Some(3u32.encode()), extrinsics: None, }, 0), ])), - (vec![100], History(vec![ + (vec![100], History::from_vec(vec![ (OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) @@ -215,13 +215,13 @@ mod test { extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, 1), ])), - (vec![101], History(vec![ + (vec![101], History::from_vec(vec![ (OverlayedValue { value: Some(vec![203]), extrinsics: Some(vec![1].into_iter().collect()) }, 0), ])), - (vec![103], History(vec![ + (vec![103], History::from_vec(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![0, 1].into_iter().collect()) @@ -304,7 +304,7 @@ mod test { let (backend, storage, mut changes) = prepare_for_build(); // 110: missing from backend, set to None in overlay - changes.changes.top.insert(vec![110], History(vec![ + changes.changes.top.insert(vec![110], History::from_vec(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![1].into_iter().collect()), diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 45c1721683f50..998d9b481f180 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -365,13 +365,13 @@ mod tests { state: 0, children: Default::default(), top: vec![ - (EXTRINSIC_INDEX.to_vec(), History(vec![ + (EXTRINSIC_INDEX.to_vec(), History::from_vec(vec![ (OverlayedValue { value: Some(3u32.encode()), extrinsics: Some(vec![1].into_iter().collect()) }, 0), ])), - (vec![1], History(vec![ + (vec![1], History::from_vec(vec![ (OverlayedValue { value: Some(vec![100]), extrinsics: Some(vec![1].into_iter().collect()) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 5102ff8ac3f8f..98c3870f60734 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -69,9 +69,23 @@ pub struct OverlayedValue { // TODO EMCH implement debug with latest vals only (at overlay changeset maybe). // this is pub crate for direct building in test (same with fields) -> TODO // make fields private and have a build method and make history private?. -#[derive(Debug, Default, Clone)] +#[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] -pub(crate) struct History(pub Vec<(V, usize)>); +pub(crate) struct History(usize, [Option<(V, usize)>; ALLOCATED_HISTORY], Vec<(V, usize)>); + +impl Default for History { + fn default() -> Self { + History( + 0, + Default::default(), + Vec::new(), + ) + } +} + +/// Size of preallocated history per element. +/// Currently at two for committed and prospective. +const ALLOCATED_HISTORY: usize = 2; /// Overlayed change set, keep history of value. #[derive(Debug, Clone)] @@ -107,18 +121,84 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { let mut result = OverlayedChangeSet::default(); - result.top = iter.into_iter().map(|(k, v)| (k, History(vec![(v, 0)]))).collect(); + result.top = iter.into_iter().map(|(k, v)| (k, { + let mut history = History::default(); + history.push(v, 0); + history + })).collect(); result } } impl History { + + #[cfg(test)] + pub fn from_vec(input: Vec<(V, usize)>) -> Self { + let mut history = History::default(); + for v in input { + history.push(v.0, v.1); + } + history + } + + fn len(&self) -> usize { + self.0 + } + + fn ix(&self, ix: usize) -> &(V, usize) { + if ix >= ALLOCATED_HISTORY { + &self.2[ix - ALLOCATED_HISTORY] + } else { + self.1[ix].as_ref().expect("Api replace unchecked [i]") + } + } + + fn ix_mut(&mut self, ix: usize) -> &mut (V, usize) { + if ix >= ALLOCATED_HISTORY { + &mut self.2[ix - ALLOCATED_HISTORY] + } else { + self.1[ix].as_mut().expect("Api replace unchecked [i]") + } + } + + fn truncate(&mut self, ix: usize) { + if ix >= ALLOCATED_HISTORY { + self.2.truncate(ix); + } + self.0 = ix; + } + + fn pop(&mut self) -> Option<(V, usize)> { + if self.0 > ALLOCATED_HISTORY { + self.0 -= 1; + self.2.pop() + } else { + if self.0 > 0 { + self.0 -= 1; + std::mem::replace(&mut self.1[self.0], None) + } else { + None + } + } + } + + // should only be call after a get_mut check + // TODO EMCH use a pair. + fn push(&mut self, val:V, tx_ix: usize) { + if self.0 < ALLOCATED_HISTORY { + self.1[self.0] = Some((val, tx_ix)); + } else { + self.2.push((val, tx_ix)); + } + self.0 += 1; + } + // TODO could have gc on get here we got some iteration that // do not need to be repeated when tx dropped (history only // updated on set). fn get(&self, history: &[TransactionState]) -> Option<&V> { // ix is never 0, - let mut ix = self.0.len(); + let mut ix = self.len(); if ix == 0 { return None; } @@ -128,12 +208,12 @@ impl History { debug_assert!(history.len() >= ix); while ix > 0 { ix -= 1; - let hix = self.0[ix].1; + let hix = self.ix(ix).1; match history[hix] { TransactionState::Pending | TransactionState::Prospective | TransactionState::Committed => - return Some(&self.0[ix].0), + return Some(&self.ix(ix).0), TransactionState::Dropped => (), } } @@ -143,18 +223,18 @@ impl History { #[cfg(test)] fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { // ix is never 0, - let mut ix = self.0.len(); + let mut ix = self.len(); if ix == 0 { return None; } debug_assert!(history.len() >= ix); while ix > 0 { ix -= 1; - let hix = self.0[ix].1; + let hix = self.ix(ix).1; match history[hix] { TransactionState::Pending | TransactionState::Prospective => - return Some(&self.0[ix].0), + return Some(&self.ix(ix).0), TransactionState::Committed | TransactionState::Dropped => (), } @@ -165,17 +245,17 @@ impl History { #[cfg(test)] fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { // ix is never 0, - let mut ix = self.0.len(); + let mut ix = self.len(); if ix == 0 { return None; } debug_assert!(history.len() >= ix); while ix > 0 { ix -= 1; - let hix = self.0[ix].1; + let hix = self.ix(ix).1; match history[hix] { TransactionState::Committed => - return Some(&self.0[ix].0), + return Some(&self.ix(ix).0), TransactionState::Pending | TransactionState::Prospective | TransactionState::Dropped => (), @@ -188,7 +268,7 @@ impl History { fn into_committed(mut self, history: &[TransactionState]) -> Option { // ix is never 0, - let mut ix = self.0.len(); + let mut ix = self.len(); if ix == 0 { return None; } @@ -198,11 +278,11 @@ impl History { debug_assert!(history.len() >= ix); while ix > 0 { ix -= 1; - let hix = self.0[ix].1; + let hix = self.ix(ix).1; match history[hix] { TransactionState::Committed => { - self.0.truncate(ix + 1); - return self.0.pop().map(|v| v.0); + self.truncate(ix + 1); + return self.pop().map(|v| v.0); }, TransactionState::Pending | TransactionState::Prospective @@ -214,7 +294,7 @@ impl History { fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { - let mut ix = self.0.len(); + let mut ix = self.len(); if ix == 0 { return None; } @@ -224,35 +304,30 @@ impl History { debug_assert!(history.len() >= ix); while ix > 0 { ix -= 1; - let hix = self.0[ix].1; + let hix = self.ix(ix).1; match history[hix] { TransactionState::Pending | TransactionState::Prospective => { - let tx = self.0[ix].1; - return Some((&mut self.0[ix].0, tx)) + let tx = self.ix(ix).1; + return Some((&mut self.ix_mut(ix).0, tx)) }, | TransactionState::Committed => { // No need for previous state if ix > 0 { - self.0.truncate(ix + 1); - let tv = self.0.pop().expect("just checked"); - self.0[0] = tv; - self.0.truncate(1); + self.truncate(ix + 1); + let tv = self.pop().expect("just checked"); + *self.ix_mut(0) = tv; + self.truncate(1); } - let tx = self.0[0].1; - return Some((&mut self.0[0].0, tx)) + let tx = self.ix(0).1; + return Some((&mut self.ix_mut(0).0, tx)) }, - TransactionState::Dropped => { let _ = self.0.pop(); }, + TransactionState::Dropped => { let _ = self.pop(); }, } } None } - // should only be call after a get_mut check - fn push(&mut self, val:V, tx_ix: usize) { - self.0.push((val, tx_ix)) - } - fn set(&mut self, history: &[TransactionState], state: usize, val: V) { // TODO EMCH : this is not optimal : can end get_mut as soon as ix < state // needs a variant for get_mut. @@ -351,7 +426,6 @@ impl OverlayedChangeSet { self.state = self.history.len() - 1; } - /// Discard a transactional layer. /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { @@ -383,7 +457,6 @@ impl OverlayedChangeSet { self.state = self.history.len() - 1; } - /// Iterator over current state of the overlay. pub fn top_iter_ext(&self) -> impl Iterator, &OverlayedValue)> { self.top.iter() @@ -391,7 +464,6 @@ impl OverlayedChangeSet { v.get(self.history.as_slice()).map(|v| (k, v))) } - /// Iterator over current state of the overlay. pub fn top_iter(&self) -> impl Iterator, Option<&Vec>)> { self.top_iter_ext().map(|(k, v)| (k, v.value.as_ref())) @@ -600,7 +672,12 @@ impl OverlayedChanges { ) -> Self { let mut changes = OverlayedChangeSet::default(); changes.top = committed.into_iter().map(|(k, v)| - (k, History(vec![(OverlayedValue { value: v, extrinsics: None }, 0)]))).collect(); + (k, { + let mut history = History::default(); + history.push(OverlayedValue { value: v, extrinsics: None }, 0); + history + }) + ).collect(); changes.commit_prospective(); let mut result = OverlayedChanges { changes, changes_trie_config: None }; prospective.into_iter().for_each(|(k, v)| result.set_storage(k, v)); @@ -649,6 +726,7 @@ pub struct BenchOverlay<'a>(pub &'a mut OverlayedChanges); #[cfg(feature = "bench")] impl<'a> BenchOverlay<'a> { + /// Call to set storage for benches. pub fn bench_set_storage(&mut self, key: Vec, val: Option>) { self.0.set_storage(key, val) } From 808a4095ca810e64ce5f1c677cb025f7dc8ed691 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 30 Jul 2019 12:54:52 +0200 Subject: [PATCH 008/129] Fix tx logic and basic tx test. --- core/state-machine/src/overlayed_changes.rs | 154 ++++++++++++++++---- 1 file changed, 122 insertions(+), 32 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 98c3870f60734..7cb523e932d00 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,19 +22,17 @@ use parity_codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -#[derive(Debug, Clone)] -#[cfg_attr(test, derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] /// State of a transaction, the state are stored /// and updated in a linear indexed way. pub(crate) enum TransactionState { /// Overlay is under change. - /// Is not necessarilly the last change. - /// Contains an index to commit position Pending, + /// A former transaction pending. + TxPending, /// Information is committed, but can still be dropped. Prospective, - /// Committed is information that cannot - /// be dropped. + /// Committed is information that cannot be dropped. Committed, /// Transaction or prospective has been dropped. Dropped, @@ -94,7 +92,7 @@ pub struct OverlayedChangeSet { /// Indexed changes history. /// First index is initial state, /// operation such as `init_transaction`, - /// `commit_transaction` or `drop_transaction` + /// `commit_transaction` or `discard_transaction` /// will change this state. pub(crate) history: Vec, /// Current state in history. @@ -183,7 +181,6 @@ impl History { } // should only be call after a get_mut check - // TODO EMCH use a pair. fn push(&mut self, val:V, tx_ix: usize) { if self.0 < ALLOCATED_HISTORY { self.1[self.0] = Some((val, tx_ix)); @@ -211,6 +208,7 @@ impl History { let hix = self.ix(ix).1; match history[hix] { TransactionState::Pending + | TransactionState::TxPending | TransactionState::Prospective | TransactionState::Committed => return Some(&self.ix(ix).0), @@ -233,6 +231,7 @@ impl History { let hix = self.ix(ix).1; match history[hix] { TransactionState::Pending + | TransactionState::TxPending | TransactionState::Prospective => return Some(&self.ix(ix).0), TransactionState::Committed @@ -257,6 +256,7 @@ impl History { TransactionState::Committed => return Some(&self.ix(ix).0), TransactionState::Pending + | TransactionState::TxPending | TransactionState::Prospective | TransactionState::Dropped => (), } @@ -264,8 +264,6 @@ impl History { None } - - fn into_committed(mut self, history: &[TransactionState]) -> Option { // ix is never 0, let mut ix = self.len(); @@ -285,6 +283,7 @@ impl History { return self.pop().map(|v| v.0); }, TransactionState::Pending + | TransactionState::TxPending | TransactionState::Prospective | TransactionState::Dropped => (), } @@ -307,21 +306,12 @@ impl History { let hix = self.ix(ix).1; match history[hix] { TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Committed | TransactionState::Prospective => { let tx = self.ix(ix).1; return Some((&mut self.ix_mut(ix).0, tx)) }, - | TransactionState::Committed => { - // No need for previous state - if ix > 0 { - self.truncate(ix + 1); - let tv = self.pop().expect("just checked"); - *self.ix_mut(0) = tv; - self.truncate(1); - } - let tx = self.ix(0).1; - return Some((&mut self.ix_mut(0).0, tx)) - }, TransactionState::Dropped => { let _ = self.pop(); }, } } @@ -400,6 +390,7 @@ impl OverlayedChangeSet { i -= 1; match self.history[i] { TransactionState::Pending + | TransactionState::TxPending | TransactionState::Prospective => self.history[i] = TransactionState::Dropped, // can have dropped from transaction -> TODO make Dropped perspective variant? = earliest // stop in some case @@ -414,15 +405,25 @@ impl OverlayedChangeSet { /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { debug_assert!(self.history.len() > 0); + let mut i = self.state + 1; + while i > 0 { + i -= 1; + match self.history[i] { + TransactionState::Prospective + | TransactionState::TxPending + | TransactionState::Pending => self.history[i] = TransactionState::Committed, + TransactionState::Dropped => (), + | TransactionState::Committed => break, + } + } // TODO EMCH can use offset and a GC state - self.history[self.state] = TransactionState::Committed; self.history.push(TransactionState::Pending); self.state = self.history.len() - 1; } - /// Create a new transactional layre + /// Create a new transactional layer pub fn start_transaction(&mut self) { - self.history.push(TransactionState::Pending); + self.history.push(TransactionState::TxPending); self.state = self.history.len() - 1; } @@ -430,19 +431,19 @@ impl OverlayedChangeSet { /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { debug_assert!(self.history.len() > self.state); - self.history[self.state] = TransactionState::Dropped; - let mut i = self.state; + let mut i = self.state + 1; // revert state to previuos pending (or create new pending) while i > 0 { i -= 1; match self.history[i] { - TransactionState::Pending => { - self.state = i; - return; + TransactionState::Prospective + | TransactionState::Pending => self.history[i] = TransactionState::Dropped, + TransactionState::TxPending => { + self.history[i] = TransactionState::Dropped; + break; }, TransactionState::Dropped => (), - TransactionState::Prospective - | TransactionState::Committed => break, + TransactionState::Committed => break, } } self.history.push(TransactionState::Pending); @@ -452,7 +453,20 @@ impl OverlayedChangeSet { /// Commit a transactional layer. pub fn commit_transaction(&mut self) { debug_assert!(self.history.len() > self.state); - self.history[self.state] = TransactionState::Prospective; + let mut i = self.state + 1; + while i > 0 { + i -= 1; + match self.history[i] { + TransactionState::Dropped => (), + TransactionState::Pending => self.history[i] = TransactionState::Prospective, + TransactionState::TxPending => { + self.history[i] = TransactionState::Prospective; + break; + }, + TransactionState::Prospective + | TransactionState::Committed => break, + } + } self.history.push(TransactionState::Pending); self.state = self.history.len() - 1; } @@ -913,4 +927,80 @@ mod tests { assert_eq!(overlay.changes.top_prospective(), Default::default()); } + + + #[test] + fn overlayed_storage_transactions() { + let mut overlayed = OverlayedChanges::default(); + + let key = vec![42, 69, 169, 142]; + + assert!(overlayed.storage(&key).is_none()); + + // discard transaction similar to discard prospective if no transaction. + + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.changes.discard_transaction(); + assert_eq!(overlayed.storage(&key), None); + + overlayed.changes.discard_prospective(); + assert_eq!(overlayed.storage(&key), None); + + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.changes.commit_transaction(); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + + overlayed.changes.discard_transaction(); + assert_eq!(overlayed.storage(&key), None); + // basic transaction test + // tx:1 + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.changes.start_transaction(); + // tx:2 + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3, 4])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3, 4][..])); + + overlayed.changes.start_transaction(); + // tx:3 + overlayed.set_storage(key.clone(), None); + assert_eq!(overlayed.storage(&key).unwrap(), None); + + overlayed.changes.discard_transaction(); + // tx:2 + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3, 4][..])); + + overlayed.changes.start_transaction(); + // tx:3 + overlayed.set_storage(key.clone(), None); + assert_eq!(overlayed.storage(&key).unwrap(), None); + + overlayed.changes.commit_transaction(); + // tx:2 + assert_eq!(overlayed.storage(&key).unwrap(), None); + + overlayed.changes.discard_transaction(); + // tx:1 + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + overlayed.changes.discard_prospective(); + assert_eq!(overlayed.storage(&key), None); + + // multiple transaction end up to prospective value + overlayed.changes.start_transaction(); + overlayed.set_storage(key.clone(), Some(vec![1])); + overlayed.changes.start_transaction(); + overlayed.set_storage(key.clone(), Some(vec![1, 2])); + overlayed.changes.start_transaction(); + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + + overlayed.changes.commit_prospective(); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + } + } From e481952171aed6fb24727c6ce20987ed1db7e630 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 30 Jul 2019 16:54:39 +0200 Subject: [PATCH 009/129] with_transaction function and high level test. --- core/executor/src/wasm_executor.rs | 12 ++++++ core/sr-api-macros/tests/runtime_calls.rs | 10 ++++- core/sr-io/src/lib.rs | 9 ++++ core/sr-io/with_std.rs | 12 ++++++ core/sr-io/without_std.rs | 36 +++++++++++++++- core/state-machine/src/basic.rs | 13 ++++++ core/state-machine/src/ext.rs | 16 +++++++ core/state-machine/src/lib.rs | 11 +++++ core/state-machine/src/overlayed_changes.rs | 48 ++++++++++++++------- core/state-machine/src/testing.rs | 13 ++++++ core/test-runtime/src/lib.rs | 14 +++++- core/test-runtime/src/system.rs | 31 ++++++++++++- srml/support/src/storage/mod.rs | 18 ++++++++ 13 files changed, 223 insertions(+), 20 deletions(-) diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index f7f60f0d5d642..64ce0343c99f4 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -1199,6 +1199,18 @@ impl_function_executor!(this: FunctionExecutor<'e, E>, this.sandbox_store.memory_teardown(memory_idx)?; Ok(()) }, + ext_start_transaction() => { + this.ext.start_transaction(); + Ok(()) + }, + ext_discard_transaction() => { + this.ext.discard_transaction(); + Ok(()) + }, + ext_commit_transaction() => { + this.ext.commit_transaction(); + Ok(()) + }, => <'e, E: Externalities + 'e> ); diff --git a/core/sr-api-macros/tests/runtime_calls.rs b/core/sr-api-macros/tests/runtime_calls.rs index 6fa155437b833..23dd30ee4ad34 100644 --- a/core/sr-api-macros/tests/runtime_calls.rs +++ b/core/sr-api-macros/tests/runtime_calls.rs @@ -102,7 +102,6 @@ fn calling_with_both_strategy_and_fail_on_native_should_work() { assert_eq!(runtime_api.fail_on_native(&block_id).unwrap(), 1); } - #[test] fn calling_with_native_else_wasm_and_faild_on_wasm_should_work() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::NativeElseWasm).build(); @@ -127,6 +126,15 @@ fn use_trie_function() { assert_eq!(runtime_api.use_trie(&block_id).unwrap(), 2); } +#[test] +fn test_transactions() { + let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); + let runtime_api = client.runtime_api(); + let block_id = BlockId::Number(client.info().chain.best_number); + assert_eq!(runtime_api.use_transactions(&block_id).unwrap(), 1); +} + + #[test] fn initialize_block_works() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 47df468d20161..bd6f06d061165 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -148,6 +148,15 @@ export_api! { /// "Commit" all existing operations and get the resultant storage change root. fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]>; + /// Start a new transaction. + fn start_transaction(); + + /// Discard a transactional layer. + fn discard_transaction(); + + /// Commit a transactional layer. + fn commit_transaction(); + /// A trie root formed from the enumerated items. /// TODO [#2382] remove (just use `ordered_trie_root` (NOTE currently not implemented for without_std)) fn enumerated_trie_root(input: &[&[u8]]) -> H::Out diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index aee2dc1bc885f..d8e43554d0636 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -164,6 +164,18 @@ impl StorageApi for () { ).unwrap_or(Ok(None)).expect("Invalid parent hash passed to storage_changes_root") } + fn start_transaction() { + ext::with(|ext| ext.start_transaction()); + } + + fn discard_transaction() { + ext::with(|ext| ext.discard_transaction()); + } + + fn commit_transaction() { + ext::with(|ext| ext.commit_transaction()); + } + fn enumerated_trie_root(input: &[&[u8]]) -> H::Out where H: Hasher, diff --git a/core/sr-io/without_std.rs b/core/sr-io/without_std.rs index ad1b26b8c1424..b463d00871181 100644 --- a/core/sr-io/without_std.rs +++ b/core/sr-io/without_std.rs @@ -90,7 +90,7 @@ pub mod ext { /// # Returns /// /// Returns the original implementation wrapped in [`RestoreImplementation`]. - pub fn replace_implementation(&'static self, new_impl: T) -> RestoreImplementation { + pub fn replace_implementation(&'static self, new_impl: T) -> RestoreImplementation { if let ExchangeableFunctionState::Replaced = self.0.get().1 { panic!("Trying to replace an already replaced implementation!") } @@ -582,7 +582,22 @@ pub mod ext { buffer_len: u32, deadline: u64 ) -> u32; + + //================================ + // Transactional Context + //================================ + + /// Open an new transactional context. + fn ext_start_transaction(); + + /// Close and dissmiss current transactional context. + fn ext_discard_transaction(); + + /// Close and commit current transactional context. + fn ext_commit_transaction(); + } + } pub use self::ext::*; @@ -763,6 +778,25 @@ impl StorageApi for () { >(_input: I) -> H::Out { unimplemented!() } + + fn start_transaction() { + unsafe { + ext_start_transaction.get()() + } + } + + fn discard_transaction() { + unsafe { + ext_discard_transaction.get()() + } + } + + fn commit_transaction() { + unsafe { + ext_commit_transaction.get()() + } + } + } impl OtherApi for () { diff --git a/core/state-machine/src/basic.rs b/core/state-machine/src/basic.rs index e9939711f1e47..b5251c1a5cb7c 100644 --- a/core/state-machine/src/basic.rs +++ b/core/state-machine/src/basic.rs @@ -160,6 +160,19 @@ impl Externalities for BasicExternalities where H::Out: Ord { warn!("Call to non-existent out offchain externalities set."); None } + + fn start_transaction(&mut self) { + self.changes.start_transaction() + } + + fn discard_transaction(&mut self) { + self.changes.discard_transaction() + } + + fn commit_transaction(&mut self) { + self.changes.commit_transaction() + } + } #[cfg(test)] diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 998d9b481f180..2b5746816b4aa 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -335,6 +335,22 @@ where fn offchain(&mut self) -> Option<&mut dyn offchain::Externalities> { self.offchain_externalities.as_mut().map(|x| &mut **x as _) } + + fn start_transaction(&mut self) { + let _guard = panic_handler::AbortGuard::new(true); + self.overlay.start_transaction() + } + + fn discard_transaction(&mut self) { + let _guard = panic_handler::AbortGuard::new(true); + self.overlay.discard_transaction() + } + + fn commit_transaction(&mut self) { + let _guard = panic_handler::AbortGuard::new(true); + self.overlay.commit_transaction() + } + } #[cfg(test)] diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index a3d1957264b1d..eb391a3477a00 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -225,6 +225,17 @@ pub trait Externalities { /// Returns offchain externalities extension if present. fn offchain(&mut self) -> Option<&mut dyn offchain::Externalities>; + + /// Create a new transactional layer. + fn start_transaction(&mut self); + + /// Discard a transactional layer, pending changes of every trasaction below this layer are + /// dropped (including committed changes) . + fn discard_transaction(&mut self); + + /// Commit a transactional layer. The changes stay attached to parent transaction layer. + fn commit_transaction(&mut self); + } /// An implementation of offchain extensions that should never be triggered. diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 7cb523e932d00..b91cd0f6948c6 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -421,7 +421,7 @@ impl OverlayedChangeSet { self.state = self.history.len() - 1; } - /// Create a new transactional layer + /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.history.push(TransactionState::TxPending); self.state = self.history.len() - 1; @@ -644,6 +644,22 @@ impl OverlayedChanges { self.changes.commit_prospective(); } + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.changes.start_transaction(); + } + + /// Discard a transactional layer. + /// A transaction is always running (history always end with pending). + pub fn discard_transaction(&mut self) { + self.changes.discard_transaction(); + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + self.changes.commit_transaction(); + } + /// Consume `OverlayedChanges` and take committed set. /// /// Panics: @@ -942,64 +958,64 @@ mod tests { overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - overlayed.changes.discard_transaction(); + overlayed.discard_transaction(); assert_eq!(overlayed.storage(&key), None); - overlayed.changes.discard_prospective(); + overlayed.discard_prospective(); assert_eq!(overlayed.storage(&key), None); overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - overlayed.changes.commit_transaction(); + overlayed.commit_transaction(); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - overlayed.changes.discard_transaction(); + overlayed.discard_transaction(); assert_eq!(overlayed.storage(&key), None); // basic transaction test // tx:1 overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - overlayed.changes.start_transaction(); + overlayed.start_transaction(); // tx:2 overlayed.set_storage(key.clone(), Some(vec![1, 2, 3, 4])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3, 4][..])); - overlayed.changes.start_transaction(); + overlayed.start_transaction(); // tx:3 overlayed.set_storage(key.clone(), None); assert_eq!(overlayed.storage(&key).unwrap(), None); - overlayed.changes.discard_transaction(); + overlayed.discard_transaction(); // tx:2 assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3, 4][..])); - overlayed.changes.start_transaction(); + overlayed.start_transaction(); // tx:3 overlayed.set_storage(key.clone(), None); assert_eq!(overlayed.storage(&key).unwrap(), None); - overlayed.changes.commit_transaction(); + overlayed.commit_transaction(); // tx:2 assert_eq!(overlayed.storage(&key).unwrap(), None); - overlayed.changes.discard_transaction(); + overlayed.discard_transaction(); // tx:1 assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - overlayed.changes.discard_prospective(); + overlayed.discard_prospective(); assert_eq!(overlayed.storage(&key), None); // multiple transaction end up to prospective value - overlayed.changes.start_transaction(); + overlayed.start_transaction(); overlayed.set_storage(key.clone(), Some(vec![1])); - overlayed.changes.start_transaction(); + overlayed.start_transaction(); overlayed.set_storage(key.clone(), Some(vec![1, 2])); - overlayed.changes.start_transaction(); + overlayed.start_transaction(); overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); - overlayed.changes.commit_prospective(); + overlayed.commit_prospective(); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); } diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 46982484be40d..b4cd4071d9939 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -232,6 +232,19 @@ impl Externalities for TestExternalities .as_mut() .map(|x| &mut **x as _) } + + fn start_transaction(&mut self) { + self.overlay.start_transaction() + } + + fn discard_transaction(&mut self) { + self.overlay.discard_transaction() + } + + fn commit_transaction(&mut self) { + self.overlay.commit_transaction() + } + } #[cfg(test)] diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index b72d2af62a9ac..93cdd22240b92 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -248,8 +248,10 @@ cfg_if! { fn function_signature_changed() -> u64; fn fail_on_native() -> u64; fn fail_on_wasm() -> u64; - /// trie no_std testing + /// Trie no_std testing. fn use_trie() -> u64; + /// Transactional tests. + fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; fn benchmark_direct_call() -> u64; /// Returns the initialized block number. @@ -281,6 +283,8 @@ cfg_if! { fn fail_on_wasm() -> u64; /// trie no_std testing fn use_trie() -> u64; + /// Transactional tests. + fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; fn benchmark_direct_call() -> u64; /// Returns the initialized block number. @@ -442,6 +446,10 @@ cfg_if! { code_using_trie() } + fn use_transactions() -> u64 { + system::test_transactions() + } + fn benchmark_indirect_call() -> u64 { let function = benchmark_add_one; (0..1000).fold(0, |p, i| p + function(i)) @@ -587,6 +595,10 @@ cfg_if! { code_using_trie() } + fn use_transactions() -> u64 { + system::test_transactions() + } + fn benchmark_indirect_call() -> u64 { (0..10000).fold(0, |p, i| p + BENCHMARK_ADD_ONE.get()(i)) } diff --git a/core/test-runtime/src/system.rs b/core/test-runtime/src/system.rs index f4433e391c7df..d5e23aab8f495 100644 --- a/core/test-runtime/src/system.rs +++ b/core/test-runtime/src/system.rs @@ -71,6 +71,35 @@ pub fn take_block_number() -> Option { Number::take() } +/// Return 1 if test successfull, 0 otherwhise. +pub fn test_transactions() -> u64 { + let origin_value: Option = storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX); + let result: Result<(), _> = storage::with_transaction(|| { + if let Ok(val) = storage::with_transaction(|| { + storage::unhashed::put(well_known_keys::EXTRINSIC_INDEX, &99u32); + if storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX) == Some(99u32) { + Ok(99u32) + } else { + Err("Error setting value will revert") + } + }) { + if storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX) == Some(val) { + Err("expected revert for test") + } else { + Err("Unexpected revert") + } + } else { + Err("Unexpected revert") + } + }); + if result == Err("expected revert for test") { + if storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX) == origin_value { + return 1; + } + } + 0 +} + #[derive(Copy, Clone)] enum Mode { Verify, @@ -206,7 +235,7 @@ pub fn finalize_block() -> Header { let mut digest = ::take().expect("StorageDigest is set by `initialize_block`"); let o_new_authorities = ::take(); - // This MUST come after all changes to storage are done. Otherwise we will fail the + // This MUST come after all changes to storage are done. Otherwise we will fail the // “Storage root does not match that calculated” assertion. let storage_root = BlakeTwo256::storage_root(); let storage_changes_root = BlakeTwo256::storage_changes_root(parent_hash); diff --git a/srml/support/src/storage/mod.rs b/srml/support/src/storage/mod.rs index a1891dade3b0c..a79be483b407e 100644 --- a/srml/support/src/storage/mod.rs +++ b/srml/support/src/storage/mod.rs @@ -27,6 +27,24 @@ pub mod storage_items; pub mod unhashed; pub mod hashed; +/// Execute under a transactional layer. +/// If the result of execution is an error, +/// the transactional layer get reverted; otherwhise +/// it is committed. +pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { + runtime_io::start_transaction(); + match f() { + Ok(r) => { + runtime_io::commit_transaction(); + Ok(r) + }, + Err(e) => { + runtime_io::discard_transaction(); + Err(e) + } + } +} + struct IncrementalInput<'a> { key: &'a [u8], pos: usize, From d9f3e502cd01f0a18b708443b972cf346c72c33b Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 30 Jul 2019 17:03:02 +0200 Subject: [PATCH 010/129] state was useless in overlayed change (allways the last ix of history). --- core/state-machine/src/changes_trie/build.rs | 1 - core/state-machine/src/ext.rs | 1 - core/state-machine/src/overlayed_changes.rs | 33 +++++++------------- 3 files changed, 11 insertions(+), 24 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 3639a561f4a3d..e1bee57f62aeb 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -196,7 +196,6 @@ mod test { changes_trie_config: Some(Configuration { digest_interval: 4, digest_levels: 2 }), changes:OverlayedChangeSet { history: vec![TransactionState::Committed, TransactionState::Pending], - state: 1, children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_vec(vec![ diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 2b5746816b4aa..237d04fc21e5e 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -378,7 +378,6 @@ mod tests { }), changes:OverlayedChangeSet { history: vec![TransactionState::Pending], - state: 0, children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_vec(vec![ diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index b91cd0f6948c6..b07395e36fea3 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -95,8 +95,6 @@ pub struct OverlayedChangeSet { /// `commit_transaction` or `discard_transaction` /// will change this state. pub(crate) history: Vec, - /// Current state in history. - pub(crate) state: usize, /// Top level storage changes. pub(crate) top: HashMap, History>, /// Child storage changes. @@ -107,7 +105,6 @@ impl Default for OverlayedChangeSet { fn default() -> Self { OverlayedChangeSet { history: vec![TransactionState::Pending], - state: 0, top: Default::default(), children: Default::default(), } @@ -336,7 +333,8 @@ impl History { } impl History { - fn set_ext(&mut self, history: &[TransactionState], state: usize, val: Option>, extrinsic_index: Option) { + fn set_ext(&mut self, history: &[TransactionState], val: Option>, extrinsic_index: Option) { + let state = history.len() - 1; if let Some((mut current, c_ix)) = self.get_mut(history) { if c_ix == state { @@ -378,14 +376,13 @@ impl OverlayedChangeSet { /// Clear the change set. pub fn clear(&mut self) { self.history = vec![TransactionState::Pending]; - self.state = 0; self.top.clear(); self.children.clear(); } /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { - let mut i = self.state + 1; + let mut i = self.history.len(); while i > 0 { i -= 1; match self.history[i] { @@ -399,13 +396,12 @@ impl OverlayedChangeSet { } } self.history.push(TransactionState::Pending); - self.state = self.history.len() - 1; } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { debug_assert!(self.history.len() > 0); - let mut i = self.state + 1; + let mut i = self.history.len(); while i > 0 { i -= 1; match self.history[i] { @@ -418,20 +414,17 @@ impl OverlayedChangeSet { } // TODO EMCH can use offset and a GC state self.history.push(TransactionState::Pending); - self.state = self.history.len() - 1; } /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.history.push(TransactionState::TxPending); - self.state = self.history.len() - 1; } /// Discard a transactional layer. /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { - debug_assert!(self.history.len() > self.state); - let mut i = self.state + 1; + let mut i = self.history.len(); // revert state to previuos pending (or create new pending) while i > 0 { i -= 1; @@ -447,13 +440,11 @@ impl OverlayedChangeSet { } } self.history.push(TransactionState::Pending); - self.state = self.history.len() - 1; } /// Commit a transactional layer. pub fn commit_transaction(&mut self) { - debug_assert!(self.history.len() > self.state); - let mut i = self.state + 1; + let mut i = self.history.len(); while i > 0 { i -= 1; match self.history[i] { @@ -468,7 +459,6 @@ impl OverlayedChangeSet { } } self.history.push(TransactionState::Pending); - self.state = self.history.len() - 1; } /// Iterator over current state of the overlay. @@ -590,7 +580,7 @@ impl OverlayedChanges { pub(crate) fn set_storage(&mut self, key: Vec, val: Option>) { let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - entry.set_ext(self.changes.history.as_slice(), self.changes.state, val, extrinsic_index); + entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -600,7 +590,7 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); - entry.set_ext(self.changes.history.as_slice(), self.changes.state, val, extrinsic_index); + entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); } /// Clear child storage of given storage key. @@ -612,10 +602,9 @@ impl OverlayedChanges { pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) { let extrinsic_index = self.extrinsic_index(); let history = self.changes.history.as_slice(); - let state = self.changes.state; let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); - map_entry.values_mut().for_each(|e| e.set_ext(history, state, None, extrinsic_index)); + map_entry.values_mut().for_each(|e| e.set_ext(history, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -629,7 +618,7 @@ impl OverlayedChanges { for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - entry.set_ext(self.changes.history.as_slice(), self.changes.state, None, extrinsic_index); + entry.set_ext(self.changes.history.as_slice(), None, extrinsic_index); } } } @@ -690,7 +679,7 @@ impl OverlayedChanges { let value = extrinsic_index.encode(); let entry = self.changes.top.entry(EXTRINSIC_INDEX.to_vec()).or_default(); - entry.set_ext(self.changes.history.as_slice(), self.changes.state, Some(value), None); + entry.set_ext(self.changes.history.as_slice(), Some(value), None); } /// Test only method to build from committed info and prospective. From cc0aa3750bef766ac9bd152a5366ca8f78e74d1f Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 30 Jul 2019 18:23:27 +0200 Subject: [PATCH 011/129] Cleaning comments and names. --- core/state-machine/src/changes_trie/build.rs | 12 +- core/state-machine/src/ext.rs | 4 +- core/state-machine/src/lib.rs | 3 +- core/state-machine/src/overlayed_changes.rs | 353 ++++++++++--------- 4 files changed, 187 insertions(+), 185 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index e1bee57f62aeb..b672c59ddab20 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -71,7 +71,7 @@ fn prepare_extrinsics_input( Number: BlockNumber, { let mut extrinsic_map = BTreeMap::, BTreeSet>::new(); - for (key, val) in changes.changes.top_iter_ext() { + for (key, val) in changes.changes.top_iter_overlay() { let extrinsics = match val.extrinsics { Some(ref extrinsics) => extrinsics, None => continue, @@ -198,13 +198,13 @@ mod test { history: vec![TransactionState::Committed, TransactionState::Pending], children: Default::default(), top: vec![ - (EXTRINSIC_INDEX.to_vec(), History::from_vec(vec![ + (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ (OverlayedValue { value: Some(3u32.encode()), extrinsics: None, }, 0), ])), - (vec![100], History::from_vec(vec![ + (vec![100], History::from_iter(vec![ (OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) @@ -214,13 +214,13 @@ mod test { extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, 1), ])), - (vec![101], History::from_vec(vec![ + (vec![101], History::from_iter(vec![ (OverlayedValue { value: Some(vec![203]), extrinsics: Some(vec![1].into_iter().collect()) }, 0), ])), - (vec![103], History::from_vec(vec![ + (vec![103], History::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![0, 1].into_iter().collect()) @@ -303,7 +303,7 @@ mod test { let (backend, storage, mut changes) = prepare_for_build(); // 110: missing from backend, set to None in overlay - changes.changes.top.insert(vec![110], History::from_vec(vec![ + changes.changes.top.insert(vec![110], History::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![1].into_iter().collect()), diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 237d04fc21e5e..21ab822ba2473 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -380,13 +380,13 @@ mod tests { history: vec![TransactionState::Pending], children: Default::default(), top: vec![ - (EXTRINSIC_INDEX.to_vec(), History::from_vec(vec![ + (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ (OverlayedValue { value: Some(3u32.encode()), extrinsics: Some(vec![1].into_iter().collect()) }, 0), ])), - (vec![1], History::from_vec(vec![ + (vec![1], History::from_iter(vec![ (OverlayedValue { value: Some(vec![100]), extrinsics: Some(vec![1].into_iter().collect()) diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index eb391a3477a00..61f91226067f6 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -972,7 +972,6 @@ where mod tests { use std::collections::HashMap; use parity_codec::Encode; - use overlayed_changes::OverlayedValue; use super::*; use super::backend::InMemory; use super::ext::Ext; @@ -1143,7 +1142,7 @@ mod tests { vec![ (b"abd".to_vec(), Some(b"69".to_vec())), (b"bbd".to_vec(), Some(b"42".to_vec())), - ]); + ], None); { let changes_trie_storage = InMemoryChangesTrieStorage::::new(); diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index b07395e36fea3..d0b42de008257 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -23,18 +23,23 @@ use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig use primitives::storage::well_known_keys::EXTRINSIC_INDEX; #[derive(Debug, Clone, PartialEq)] -/// State of a transaction, the state are stored -/// and updated in a linear indexed way. +/// State of a transaction, the states are stored +/// in a linear indexed history. pub(crate) enum TransactionState { - /// Overlay is under change. + /// Overlay is under change and can still be dropped. Pending, - /// A former transaction pending. + /// Overlay is under change and can still be dropped. + /// This also mark the start of a transaction. TxPending, - /// Information is committed, but can still be dropped. + /// Information is committed, but can still be dropped + /// from `discard_prospective` or `discard_transaction` + /// of a parent transaction. Prospective, /// Committed is information that cannot be dropped. Committed, /// Transaction or prospective has been dropped. + /// Information pointing to this indexed historic state should + /// not be returned and can be removed. Dropped, } @@ -44,8 +49,6 @@ pub(crate) enum TransactionState { /// that can be cleared. #[derive(Debug, Default, Clone)] pub struct OverlayedChanges { -// /// History current position. -// pub(crate) current: usize, -> end of history vec /// Changes with their history. pub(crate) changes: OverlayedChangeSet, /// Changes trie configuration. None by default, but could be installed by the @@ -57,16 +60,19 @@ pub struct OverlayedChanges { #[derive(Debug, Default, Clone)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedValue { - /// Values change historic with history state index. + /// Current value. None if value has been deleted. pub value: Option>, /// The set of extinsic indices where the values has been changed. - /// Same history system as value (but over . pub extrinsics: Option>, } -// TODO EMCH implement debug with latest vals only (at overlay changeset maybe). -// this is pub crate for direct building in test (same with fields) -> TODO -// make fields private and have a build method and make history private?. +/// History of value that are related to a state history (eg field `history` of +/// an `OverlayedChangeSet`). +/// +/// First field is the current size of the history. +/// Second field are statically allocated values for performance purpose. +/// Third field are dynamically allocated values. +/// Values are always paired with a state history index. #[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] pub(crate) struct History(usize, [Option<(V, usize)>; ALLOCATED_HISTORY], Vec<(V, usize)>); @@ -82,18 +88,22 @@ impl Default for History { } /// Size of preallocated history per element. -/// Currently at two for committed and prospective. +/// Currently at two for committed and prospective only. +/// It means that using transaction in a module got a direct allocation cost. const ALLOCATED_HISTORY: usize = 2; -/// Overlayed change set, keep history of value. +/// Overlayed change set, keep history of values. +/// +/// This does not work by stacking hashmap for transaction, +/// but store history of value instead. +/// Value validity is given by a global indexed state history. +/// When dropping or committing a layer of transaction, +/// history for each values is kept until +/// next mutable access to the value. #[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { - /// Indexed changes history. - /// First index is initial state, - /// operation such as `init_transaction`, - /// `commit_transaction` or `discard_transaction` - /// will change this state. + /// Indexed state history. pub(crate) history: Vec, /// Top level storage changes. pub(crate) top: HashMap, History>, @@ -125,10 +135,32 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { } } +impl std::ops::Index for History { + type Output = (V, usize); + fn index(&self, index: usize) -> &Self::Output { + if index >= ALLOCATED_HISTORY { + &self.2[index - ALLOCATED_HISTORY] + } else { + self.1[index].as_ref().expect("Index api fail on non existing value") + } + } +} + +impl std::ops::IndexMut for History { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + if index >= ALLOCATED_HISTORY { + &mut self.2[index - ALLOCATED_HISTORY] + } else { + self.1[index].as_mut().expect("Index api fail on non existing value") + } + } +} + impl History { #[cfg(test)] - pub fn from_vec(input: Vec<(V, usize)>) -> Self { + /// Create an history from an existing history. + pub fn from_iter(input: impl IntoIterator) -> Self { let mut history = History::default(); for v in input { history.push(v.0, v.1); @@ -140,27 +172,11 @@ impl History { self.0 } - fn ix(&self, ix: usize) -> &(V, usize) { - if ix >= ALLOCATED_HISTORY { - &self.2[ix - ALLOCATED_HISTORY] - } else { - self.1[ix].as_ref().expect("Api replace unchecked [i]") - } - } - - fn ix_mut(&mut self, ix: usize) -> &mut (V, usize) { - if ix >= ALLOCATED_HISTORY { - &mut self.2[ix - ALLOCATED_HISTORY] - } else { - self.1[ix].as_mut().expect("Api replace unchecked [i]") - } - } - - fn truncate(&mut self, ix: usize) { - if ix >= ALLOCATED_HISTORY { - self.2.truncate(ix); + fn truncate(&mut self, index: usize) { + if index >= ALLOCATED_HISTORY { + self.2.truncate(index); } - self.0 = ix; + self.0 = index; } fn pop(&mut self) -> Option<(V, usize)> { @@ -178,38 +194,34 @@ impl History { } // should only be call after a get_mut check - fn push(&mut self, val:V, tx_ix: usize) { + fn push(&mut self, val: V, tx_index: usize) { if self.0 < ALLOCATED_HISTORY { - self.1[self.0] = Some((val, tx_ix)); + self.1[self.0] = Some((val, tx_index)); } else { - self.2.push((val, tx_ix)); + self.2.push((val, tx_index)); } self.0 += 1; } - // TODO could have gc on get here we got some iteration that - // do not need to be repeated when tx dropped (history only - // updated on set). + /// When possible please prefer `get_mut` as it will free + /// memory. fn get(&self, history: &[TransactionState]) -> Option<&V> { - // ix is never 0, - let mut ix = self.len(); - if ix == 0 { + // index is never 0, + let mut index = self.len(); + if index == 0 { return None; } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= ix); - while ix > 0 { - ix -= 1; - let hix = self.ix(ix).1; - match history[hix] { + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective | TransactionState::Committed => - return Some(&self.ix(ix).0), - TransactionState::Dropped => (), + return Some(&self[index].0), } } None @@ -217,20 +229,20 @@ impl History { #[cfg(test)] fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { - // ix is never 0, - let mut ix = self.len(); - if ix == 0 { + // index is never 0, + let mut index = self.len(); + if index == 0 { return None; } - debug_assert!(history.len() >= ix); - while ix > 0 { - ix -= 1; - let hix = self.ix(ix).1; - match history[hix] { + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective => - return Some(&self.ix(ix).0), + return Some(&self[index].0), TransactionState::Committed | TransactionState::Dropped => (), } @@ -240,18 +252,18 @@ impl History { #[cfg(test)] fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { - // ix is never 0, - let mut ix = self.len(); - if ix == 0 { + // index is never 0, + let mut index = self.len(); + if index == 0 { return None; } - debug_assert!(history.len() >= ix); - while ix > 0 { - ix -= 1; - let hix = self.ix(ix).1; - match history[hix] { + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { TransactionState::Committed => - return Some(&self.ix(ix).0), + return Some(&self[index].0), TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective @@ -262,21 +274,21 @@ impl History { } fn into_committed(mut self, history: &[TransactionState]) -> Option { - // ix is never 0, - let mut ix = self.len(); - if ix == 0 { + // index is never 0, + let mut index = self.len(); + if index == 0 { return None; } // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= ix); - while ix > 0 { - ix -= 1; - let hix = self.ix(ix).1; - match history[hix] { + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { TransactionState::Committed => { - self.truncate(ix + 1); + self.truncate(index + 1); return self.pop().map(|v| v.0); }, TransactionState::Pending @@ -290,24 +302,24 @@ impl History { fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { - let mut ix = self.len(); - if ix == 0 { + let mut index = self.len(); + if index == 0 { return None; } // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= ix); - while ix > 0 { - ix -= 1; - let hix = self.ix(ix).1; - match history[hix] { + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { TransactionState::Pending | TransactionState::TxPending | TransactionState::Committed | TransactionState::Prospective => { - let tx = self.ix(ix).1; - return Some((&mut self.ix_mut(ix).0, tx)) + let tx = self[index].1; + return Some((&mut self[index].0, tx)) }, TransactionState::Dropped => { let _ = self.pop(); }, } @@ -315,12 +327,13 @@ impl History { None } - fn set(&mut self, history: &[TransactionState], state: usize, val: V) { - // TODO EMCH : this is not optimal : can end get_mut as soon as ix < state + fn set(&mut self, history: &[TransactionState], val: V) { + // TODO EMCH : this is not optimal : can end get_mut as soon as index < state // needs a variant for get_mut. match self.get_mut(history) { - Some((v, ix)) => { - if ix == state { + Some((v, index)) => { + let state = history.len() - 1; + if index == state { *v = val; return; } @@ -333,37 +346,54 @@ impl History { } impl History { - fn set_ext(&mut self, history: &[TransactionState], val: Option>, extrinsic_index: Option) { - let state = history.len() - 1; - if let Some((mut current, c_ix)) = self.get_mut(history) { - - if c_ix == state { - current.value = val; - if let Some(extrinsic) = extrinsic_index { - current.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } - - } else { - let mut extrinsics = current.extrinsics.clone(); - extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic)); - self.push(OverlayedValue { - value: val, - extrinsics, - }, state); - } - } else { - let mut extrinsics: Option> = None; - extrinsic_index.map(|extrinsic| extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic)); + fn set_with_extrinsic( + &mut self, history: &[TransactionState], + val: Option>, + extrinsic_index: Option, + ) { + if let Some(extrinsic) = extrinsic_index { + self.set_with_extrinsic_inner(history, val, extrinsic) + } else { + self.set(history, OverlayedValue { + value: val, + extrinsics: None, + }) + } + } + + fn set_with_extrinsic_inner( + &mut self, history: &[TransactionState], + val: Option>, + extrinsic_index: u32, + ) { + let state = history.len() - 1; + if let Some((mut current, current_index)) = self.get_mut(history) { + if current_index == state { + current.value = val; + current.extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); + + } else { + let mut extrinsics = current.extrinsics.clone(); + extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); self.push(OverlayedValue { - value: val, - extrinsics, - }, state); - + value: val, + extrinsics, + }, state); } + } else { + let mut extrinsics: Option> = None; + extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); + + self.push(OverlayedValue { + value: val, + extrinsics, + }, state); + + } } } @@ -373,25 +403,16 @@ impl OverlayedChangeSet { self.top.is_empty() && self.children.is_empty() } - /// Clear the change set. - pub fn clear(&mut self) { - self.history = vec![TransactionState::Pending]; - self.top.clear(); - self.children.clear(); - } - /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { let mut i = self.history.len(); while i > 0 { i -= 1; match self.history[i] { + TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective => self.history[i] = TransactionState::Dropped, - // can have dropped from transaction -> TODO make Dropped perspective variant? = earliest - // stop in some case - TransactionState::Dropped => (), TransactionState::Committed => break, } } @@ -405,14 +426,13 @@ impl OverlayedChangeSet { while i > 0 { i -= 1; match self.history[i] { + TransactionState::Dropped => (), TransactionState::Prospective | TransactionState::TxPending | TransactionState::Pending => self.history[i] = TransactionState::Committed, - TransactionState::Dropped => (), | TransactionState::Committed => break, } } - // TODO EMCH can use offset and a GC state self.history.push(TransactionState::Pending); } @@ -429,13 +449,13 @@ impl OverlayedChangeSet { while i > 0 { i -= 1; match self.history[i] { + TransactionState::Dropped => (), TransactionState::Prospective | TransactionState::Pending => self.history[i] = TransactionState::Dropped, TransactionState::TxPending => { self.history[i] = TransactionState::Dropped; break; }, - TransactionState::Dropped => (), TransactionState::Committed => break, } } @@ -462,7 +482,7 @@ impl OverlayedChangeSet { } /// Iterator over current state of the overlay. - pub fn top_iter_ext(&self) -> impl Iterator, &OverlayedValue)> { + pub fn top_iter_overlay(&self) -> impl Iterator, &OverlayedValue)> { self.top.iter() .filter_map(move |(k, v)| v.get(self.history.as_slice()).map(|v| (k, v))) @@ -470,11 +490,11 @@ impl OverlayedChangeSet { /// Iterator over current state of the overlay. pub fn top_iter(&self) -> impl Iterator, Option<&Vec>)> { - self.top_iter_ext().map(|(k, v)| (k, v.value.as_ref())) + self.top_iter_overlay().map(|(k, v)| (k, v.value.as_ref())) } /// Iterator over current state of the overlay. - pub fn child_iter_ext( + pub fn child_iter_overlay( &self, storage_key: &[u8], ) -> impl Iterator, &OverlayedValue)> { @@ -488,7 +508,7 @@ impl OverlayedChangeSet { /// Iterator over current state of the overlay. pub fn child_iter(&self, storage_key: &[u8]) -> impl Iterator, Option<&Vec>)> { - self.child_iter_ext(storage_key).map(|(k, v)| (k, v.value.as_ref())) + self.child_iter_overlay(storage_key).map(|(k, v)| (k, v.value.as_ref())) } /// Test only method to access current prospective changes. @@ -541,8 +561,9 @@ impl OverlayedChanges { self.changes_trie_config = Some(config); true } - /// To make some non followed change + #[cfg(test)] + /// To allow value without extrinsic this can be use in test to disable change trie. pub(crate) fn remove_changes_trie_config(&mut self) -> Option { self.changes_trie_config.take() } @@ -580,7 +601,7 @@ impl OverlayedChanges { pub(crate) fn set_storage(&mut self, key: Vec, val: Option>) { let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); + entry.set_with_extrinsic(self.changes.history.as_slice(), val, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -590,7 +611,7 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); - entry.set_ext(self.changes.history.as_slice(), val, extrinsic_index); + entry.set_with_extrinsic(self.changes.history.as_slice(), val, extrinsic_index); } /// Clear child storage of given storage key. @@ -604,7 +625,7 @@ impl OverlayedChanges { let history = self.changes.history.as_slice(); let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); - map_entry.values_mut().for_each(|e| e.set_ext(history, None, extrinsic_index)); + map_entry.values_mut().for_each(|e| e.set_with_extrinsic(history, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -618,7 +639,7 @@ impl OverlayedChanges { for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - entry.set_ext(self.changes.history.as_slice(), None, extrinsic_index); + entry.set_with_extrinsic(self.changes.history.as_slice(), None, extrinsic_index); } } } @@ -665,9 +686,9 @@ impl OverlayedChanges { top.into_iter() .filter_map(move |(k, v)| v.into_committed(history.as_slice()).map(|v| (k, v.value))), children.into_iter().map(move |(sk, v)| { - let historys = history2.clone(); + let history2 = history2.clone(); (sk, v.into_iter() - .filter_map(move |(k, v)| v.into_committed(historys.as_slice()).map(|v| (k, v.value)))) + .filter_map(move |(k, v)| v.into_committed(history2.as_slice()).map(|v| (k, v.value)))) }) ) } @@ -679,7 +700,10 @@ impl OverlayedChanges { let value = extrinsic_index.encode(); let entry = self.changes.top.entry(EXTRINSIC_INDEX.to_vec()).or_default(); - entry.set_ext(self.changes.history.as_slice(), Some(value), None); + entry.set(self.changes.history.as_slice(), OverlayedValue{ + value: Some(value), + extrinsics: None, + }); } /// Test only method to build from committed info and prospective. @@ -688,30 +712,9 @@ impl OverlayedChanges { pub(crate) fn new_from_top( committed: Vec<(Vec, Option>)>, prospective: Vec<(Vec, Option>)>, - ) -> Self { - let mut changes = OverlayedChangeSet::default(); - changes.top = committed.into_iter().map(|(k, v)| - (k, { - let mut history = History::default(); - history.push(OverlayedValue { value: v, extrinsics: None }, 0); - history - }) - ).collect(); - changes.commit_prospective(); - let mut result = OverlayedChanges { changes, changes_trie_config: None }; - prospective.into_iter().for_each(|(k, v)| result.set_storage(k, v)); - result - } - /// Test only method to build from committed info and prospective. - /// Create an history of two states. - /// TODO EMCH replace prev fn with this one. - #[cfg(test)] - pub(crate) fn new_from_top_changes( - committed: Vec<(Vec, Option>)>, - prospective: Vec<(Vec, Option>)>, changes_trie_config: Option, ) -> Self { - let mut changes = OverlayedChangeSet::default(); + let changes = OverlayedChangeSet::default(); let mut result = OverlayedChanges { changes, changes_trie_config }; committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); result.changes.commit_prospective(); @@ -820,7 +823,7 @@ mod tests { vec![ (b"dogglesworth".to_vec(), Some(b"cat".to_vec()).into()), (b"doug".to_vec(), None.into()), - ].into_iter().collect()); + ].into_iter().collect(), None); let changes_trie_storage = InMemoryChangesTrieStorage::::new(); let mut ext = Ext::new( From 0c8235e4dfa13b81ac344396316d26d2a641eaf2 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 31 Jul 2019 10:22:52 +0200 Subject: [PATCH 012/129] Rename Externalities primitives to 'storage_' + old name. Remove bench feature, exposing 'set_storage' publicly. Minor fixes. --- core/executor/src/wasm_executor.rs | 20 ++++++++--------- core/sr-io/src/lib.rs | 6 +++--- core/sr-io/with_std.rs | 12 +++++------ core/sr-io/without_std.rs | 24 ++++++++++----------- core/state-machine/Cargo.toml | 4 ---- core/state-machine/benches/bench.rs | 13 ++++++----- core/state-machine/src/basic.rs | 10 ++++----- core/state-machine/src/ext.rs | 8 +++---- core/state-machine/src/lib.rs | 8 +++---- core/state-machine/src/overlayed_changes.rs | 15 +------------ core/state-machine/src/testing.rs | 6 +++--- srml/support/src/storage/mod.rs | 9 ++++---- 12 files changed, 58 insertions(+), 77 deletions(-) diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index 1195e7e61b6c5..3a9340979888a 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -1247,17 +1247,17 @@ impl_function_executor!(this: FunctionExecutor<'e, E>, this.sandbox_store.memory_teardown(memory_idx)?; Ok(()) }, - ext_start_transaction() => { - this.ext.start_transaction(); - Ok(()) - }, - ext_discard_transaction() => { - this.ext.discard_transaction(); - Ok(()) + ext_storage_start_transaction() => { + this.ext.storage_start_transaction(); + Ok(()) + }, + ext_storage_discard_transaction() => { + this.ext.storage_discard_transaction(); + Ok(()) }, - ext_commit_transaction() => { - this.ext.commit_transaction(); - Ok(()) + ext_storage_commit_transaction() => { + this.ext.storage_commit_transaction(); + Ok(()) }, => <'e, E: Externalities + 'e> ); diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index a3fa598d33bd9..a8b109a416acd 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -152,13 +152,13 @@ export_api! { fn storage_changes_root(parent_hash: [u8; 32]) -> Option<[u8; 32]>; /// Start a new transaction. - fn start_transaction(); + fn storage_start_transaction(); /// Discard a transactional layer. - fn discard_transaction(); + fn storage_discard_transaction(); /// Commit a transactional layer. - fn commit_transaction(); + fn storage_commit_transaction(); /// A trie root formed from the enumerated items. /// TODO [#2382] remove (just use `ordered_trie_root` (NOTE currently not implemented for without_std)) diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index 0b7423889a11f..bb524a1a83aa4 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -164,16 +164,16 @@ impl StorageApi for () { ).unwrap_or(Ok(None)).expect("Invalid parent hash passed to storage_changes_root") } - fn start_transaction() { - ext::with(|ext| ext.start_transaction()); + fn storage_start_transaction() { + ext::with(|ext| ext.storage_start_transaction()); } - fn discard_transaction() { - ext::with(|ext| ext.discard_transaction()); + fn storage_discard_transaction() { + ext::with(|ext| ext.storage_discard_transaction()); } - fn commit_transaction() { - ext::with(|ext| ext.commit_transaction()); + fn storage_commit_transaction() { + ext::with(|ext| ext.storage_commit_transaction()); } fn enumerated_trie_root(input: &[&[u8]]) -> H::Out diff --git a/core/sr-io/without_std.rs b/core/sr-io/without_std.rs index 7d0560b2233fc..70b68cda78b0c 100644 --- a/core/sr-io/without_std.rs +++ b/core/sr-io/without_std.rs @@ -629,14 +629,14 @@ pub mod ext { // Transactional Context //================================ - /// Open an new transactional context. - fn ext_start_transaction(); + /// Open a new transactional storage layer. + fn ext_storage_start_transaction(); - /// Close and dissmiss current transactional context. - fn ext_discard_transaction(); + /// Close and dismiss current transactional storage layer. + fn ext_storage_discard_transaction(); - /// Close and commit current transactional context. - fn ext_commit_transaction(); + /// Close and commit current transactional storage layer. + fn ext_storage_commit_transaction(); } @@ -821,21 +821,21 @@ impl StorageApi for () { unimplemented!() } - fn start_transaction() { + fn storage_start_transaction() { unsafe { - ext_start_transaction.get()() + ext_storage_start_transaction.get()() } } - fn discard_transaction() { + fn storage_discard_transaction() { unsafe { - ext_discard_transaction.get()() + ext_storage_discard_transaction.get()() } } - fn commit_transaction() { + fn storage_commit_transaction() { unsafe { - ext_commit_transaction.get()() + ext_storage_commit_transaction.get()() } } diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index 670a777364e31..0cb77a86777f3 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -25,7 +25,3 @@ rand = "0.6" [[bench]] name = "bench" harness = false - -[features] -default = ["bench"] -bench = [] diff --git a/core/state-machine/benches/bench.rs b/core/state-machine/benches/bench.rs index ffae896c84251..38f7c21520279 100644 --- a/core/state-machine/benches/bench.rs +++ b/core/state-machine/benches/bench.rs @@ -17,7 +17,6 @@ use criterion::{Criterion, black_box, Bencher}; use criterion::{criterion_group, criterion_main}; use substrate_state_machine::OverlayedChanges; -use substrate_state_machine::BenchOverlay; const CONTENT_KEY_SIZE: usize = 32; @@ -55,15 +54,15 @@ fn commit_drop_commit(b: &mut Bencher, input: &Vec) { b.iter(move || { let mut overlayed = OverlayedChanges::default(); for i in key_vals.iter() { - BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); } overlayed.commit_prospective(); for i in key_vals.iter() { - BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); } overlayed.discard_prospective(); for i in key_vals.iter() { - BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); } overlayed.commit_prospective(); }); @@ -74,14 +73,14 @@ fn commit_drop_commit_and_get(b: &mut Bencher, input: &Vec) { b.iter(move || { let mut overlayed = OverlayedChanges::default(); for i in key_vals.iter() { - BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); } for i in key_vals.iter() { black_box(overlayed.storage(&i.0)); } overlayed.commit_prospective(); for i in key_vals.iter() { - BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); } for i in key_vals.iter() { black_box(overlayed.storage(&i.0)); @@ -91,7 +90,7 @@ fn commit_drop_commit_and_get(b: &mut Bencher, input: &Vec) { black_box(overlayed.storage(&i.0)); } for i in key_vals.iter() { - BenchOverlay(&mut overlayed).bench_set_storage(i.0.clone(), Some(i.1.clone())); + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); } overlayed.commit_prospective(); }); diff --git a/core/state-machine/src/basic.rs b/core/state-machine/src/basic.rs index f91fb74d32f21..25d97584cd7df 100644 --- a/core/state-machine/src/basic.rs +++ b/core/state-machine/src/basic.rs @@ -173,15 +173,15 @@ impl Externalities for BasicExternalities where H::Out: Ord { None } - fn start_transaction(&mut self) { - warn!("No support for transaction"); + fn storage_start_transaction(&mut self) { + warn!("No support for storage transaction"); } - fn discard_transaction(&mut self) { - panic!("No support for transaction"); + fn storage_discard_transaction(&mut self) { + panic!("No support for storage transaction"); } - fn commit_transaction(&mut self) { + fn storage_commit_transaction(&mut self) { // no need to fail in this case } diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 6ff9056186d93..2e3d751eb303a 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -322,17 +322,17 @@ where self.offchain_externalities.as_mut().map(|x| &mut **x as _) } - fn start_transaction(&mut self) { + fn storage_start_transaction(&mut self) { let _guard = panic_handler::AbortGuard::new(true); self.overlay.start_transaction() } - fn discard_transaction(&mut self) { + fn storage_discard_transaction(&mut self) { let _guard = panic_handler::AbortGuard::new(true); self.overlay.discard_transaction() } - fn commit_transaction(&mut self) { + fn storage_commit_transaction(&mut self) { let _guard = panic_handler::AbortGuard::new(true); self.overlay.commit_transaction() } @@ -362,7 +362,7 @@ mod tests { digest_interval: 0, digest_levels: 0, }), - changes:OverlayedChangeSet { + changes: OverlayedChangeSet { history: vec![TransactionState::Pending], children: Default::default(), top: vec![ diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 8d2b42582b7f0..52dd8cddaad8e 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -53,8 +53,6 @@ pub use changes_trie::{ oldest_non_pruned_trie as oldest_non_pruned_changes_trie }; pub use overlayed_changes::OverlayedChanges; -#[cfg(feature = "bench")] -pub use overlayed_changes::BenchOverlay; pub use proving_backend::{ create_proof_check_backend, create_proof_check_backend_storage, Recorder as ProofRecorder, ProvingBackend, @@ -227,14 +225,14 @@ pub trait Externalities { fn offchain(&mut self) -> Option<&mut dyn offchain::Externalities>; /// Create a new transactional layer. - fn start_transaction(&mut self); + fn storage_start_transaction(&mut self); /// Discard a transactional layer, pending changes of every trasaction below this layer are /// dropped (including committed changes) . - fn discard_transaction(&mut self); + fn storage_discard_transaction(&mut self); /// Commit a transactional layer. The changes stay attached to parent transaction layer. - fn commit_transaction(&mut self); + fn storage_commit_transaction(&mut self); } diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 8d4e5e58d3e14..3b0cb8a39991e 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -610,7 +610,7 @@ impl OverlayedChanges { /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_storage(&mut self, key: Vec, val: Option>) { + pub fn set_storage(&mut self, key: Vec, val: Option>) { let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); entry.set_with_extrinsic(self.changes.history.as_slice(), val, extrinsic_index); @@ -753,19 +753,6 @@ impl OverlayedChanges { } } -#[cfg(feature = "bench")] -/// Expose private function of overlay for benching. -pub struct BenchOverlay<'a>(pub &'a mut OverlayedChanges); - - -#[cfg(feature = "bench")] -impl<'a> BenchOverlay<'a> { - /// Call to set storage for benches. - pub fn bench_set_storage(&mut self, key: Vec, val: Option>) { - self.0.set_storage(key, val) - } -} - #[cfg(test)] impl From>> for OverlayedValue { fn from(value: Option>) -> OverlayedValue { diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 660591224d989..8af2f6d14c3d0 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -255,15 +255,15 @@ impl Externalities for TestExternalities .map(|x| &mut **x as _) } - fn start_transaction(&mut self) { + fn storage_start_transaction(&mut self) { self.overlay.start_transaction() } - fn discard_transaction(&mut self) { + fn storage_discard_transaction(&mut self) { self.overlay.discard_transaction() } - fn commit_transaction(&mut self) { + fn storage_commit_transaction(&mut self) { self.overlay.commit_transaction() } diff --git a/srml/support/src/storage/mod.rs b/srml/support/src/storage/mod.rs index 614af5c1c4ebb..22035798c4502 100644 --- a/srml/support/src/storage/mod.rs +++ b/srml/support/src/storage/mod.rs @@ -28,18 +28,19 @@ pub mod unhashed; pub mod hashed; /// Execute under a transactional layer. +/// /// If the result of execution is an error, /// the transactional layer get reverted; otherwhise /// it is committed. -pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { - runtime_io::start_transaction(); +pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { + runtime_io::storage_start_transaction(); match f() { Ok(r) => { - runtime_io::commit_transaction(); + runtime_io::storage_commit_transaction(); Ok(r) }, Err(e) => { - runtime_io::discard_transaction(); + runtime_io::storage_discard_transaction(); Err(e) } } From cade913a54f97cc6f7738137684d0a5ca0c9fe47 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 31 Jul 2019 14:54:21 +0200 Subject: [PATCH 013/129] name fields instead of tuble. --- core/state-machine/src/overlayed_changes.rs | 50 +++++++++++---------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 3b0cb8a39991e..a9684d4fc58e7 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -76,15 +76,19 @@ pub struct OverlayedValue { /// Values are always paired with a state history index. #[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] -pub(crate) struct History(usize, [Option<(V, usize)>; ALLOCATED_HISTORY], Vec<(V, usize)>); +pub(crate) struct History { + length: usize, + static_content: [Option<(V, usize)>; ALLOCATED_HISTORY], + dyn_content: Vec<(V, usize)> +} impl Default for History { fn default() -> Self { - History( - 0, - Default::default(), - Vec::new(), - ) + History { + length: 0, + static_content: Default::default(), + dyn_content: Vec::new(), + } } } @@ -140,9 +144,9 @@ impl std::ops::Index for History { type Output = (V, usize); fn index(&self, index: usize) -> &Self::Output { if index >= ALLOCATED_HISTORY { - &self.2[index - ALLOCATED_HISTORY] + &self.dyn_content[index - ALLOCATED_HISTORY] } else { - self.1[index].as_ref().expect("Index api fail on non existing value") + self.static_content[index].as_ref().expect("Index api fail on non existing value") } } } @@ -150,9 +154,9 @@ impl std::ops::Index for History { impl std::ops::IndexMut for History { fn index_mut(&mut self, index: usize) -> &mut Self::Output { if index >= ALLOCATED_HISTORY { - &mut self.2[index - ALLOCATED_HISTORY] + &mut self.dyn_content[index - ALLOCATED_HISTORY] } else { - self.1[index].as_mut().expect("Index api fail on non existing value") + self.static_content[index].as_mut().expect("Index api fail on non existing value") } } } @@ -170,24 +174,24 @@ impl History { } fn len(&self) -> usize { - self.0 + self.length } fn truncate(&mut self, index: usize) { if index >= ALLOCATED_HISTORY { - self.2.truncate(index); + self.dyn_content.truncate(index); } - self.0 = index; + self.length = index; } fn pop(&mut self) -> Option<(V, usize)> { - if self.0 > ALLOCATED_HISTORY { - self.0 -= 1; - self.2.pop() + if self.length > ALLOCATED_HISTORY { + self.length -= 1; + self.dyn_content.pop() } else { - if self.0 > 0 { - self.0 -= 1; - std::mem::replace(&mut self.1[self.0], None) + if self.length > 0 { + self.length -= 1; + std::mem::replace(&mut self.static_content[self.length], None) } else { None } @@ -196,12 +200,12 @@ impl History { // should only be call after a get_mut check fn push(&mut self, val: V, tx_index: usize) { - if self.0 < ALLOCATED_HISTORY { - self.1[self.0] = Some((val, tx_index)); + if self.length < ALLOCATED_HISTORY { + self.static_content[self.length] = Some((val, tx_index)); } else { - self.2.push((val, tx_index)); + self.dyn_content.push((val, tx_index)); } - self.0 += 1; + self.length += 1; } /// When possible please prefer `get_mut` as it will free From fd26b2ad7d2661b217b1f11ed6c1e4b2695910a6 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 31 Jul 2019 15:15:41 +0200 Subject: [PATCH 014/129] Update srml/support/src/storage/mod.rs Co-Authored-By: Sergei Pepyakin --- srml/support/src/storage/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/srml/support/src/storage/mod.rs b/srml/support/src/storage/mod.rs index 22035798c4502..442072de3d284 100644 --- a/srml/support/src/storage/mod.rs +++ b/srml/support/src/storage/mod.rs @@ -29,6 +29,7 @@ pub mod hashed; /// Execute under a transactional layer. /// +/// /// If the result of execution is an error, /// the transactional layer get reverted; otherwhise /// it is committed. From 32459683e0ac29370c39d99a68f8f8aac8306614 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 1 Aug 2019 09:15:58 +0200 Subject: [PATCH 015/129] Use more efficient smallvec. --- Cargo.lock | 1 + core/state-machine/Cargo.toml | 1 + core/state-machine/src/overlayed_changes.rs | 51 ++++----------------- 3 files changed, 11 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dac5ab56ea2ca..a1d629cd4ad6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4819,6 +4819,7 @@ dependencies = [ "parity-codec 4.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-panic-handler 2.0.0", "substrate-primitives 2.0.0", "substrate-trie 2.0.0", diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index 0cb77a86777f3..905baebee3ebb 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -16,6 +16,7 @@ primitives = { package = "substrate-primitives", path = "../primitives" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } parity-codec = "4.1.1" num-traits = "0.2" +smallvec = "0.6" [dev-dependencies] hex-literal = "0.2.0" diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index a9684d4fc58e7..8bde8a0ff01d0 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -21,6 +21,7 @@ use std::collections::{HashMap, BTreeSet}; use parity_codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; +use smallvec::SmallVec; #[derive(Debug, Clone, PartialEq)] /// State of a transaction, the states are stored @@ -76,19 +77,11 @@ pub struct OverlayedValue { /// Values are always paired with a state history index. #[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] -pub(crate) struct History { - length: usize, - static_content: [Option<(V, usize)>; ALLOCATED_HISTORY], - dyn_content: Vec<(V, usize)> -} +pub(crate) struct History (SmallVec<[(V, usize); ALLOCATED_HISTORY]>); impl Default for History { fn default() -> Self { - History { - length: 0, - static_content: Default::default(), - dyn_content: Vec::new(), - } + History(SmallVec::new()) } } @@ -143,21 +136,13 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { impl std::ops::Index for History { type Output = (V, usize); fn index(&self, index: usize) -> &Self::Output { - if index >= ALLOCATED_HISTORY { - &self.dyn_content[index - ALLOCATED_HISTORY] - } else { - self.static_content[index].as_ref().expect("Index api fail on non existing value") - } + self.0.index(index) } } impl std::ops::IndexMut for History { fn index_mut(&mut self, index: usize) -> &mut Self::Output { - if index >= ALLOCATED_HISTORY { - &mut self.dyn_content[index - ALLOCATED_HISTORY] - } else { - self.static_content[index].as_mut().expect("Index api fail on non existing value") - } + self.0.index_mut(index) } } @@ -174,38 +159,20 @@ impl History { } fn len(&self) -> usize { - self.length + self.0.len() } fn truncate(&mut self, index: usize) { - if index >= ALLOCATED_HISTORY { - self.dyn_content.truncate(index); - } - self.length = index; + self.0.truncate(index) } fn pop(&mut self) -> Option<(V, usize)> { - if self.length > ALLOCATED_HISTORY { - self.length -= 1; - self.dyn_content.pop() - } else { - if self.length > 0 { - self.length -= 1; - std::mem::replace(&mut self.static_content[self.length], None) - } else { - None - } - } + self.0.pop() } // should only be call after a get_mut check fn push(&mut self, val: V, tx_index: usize) { - if self.length < ALLOCATED_HISTORY { - self.static_content[self.length] = Some((val, tx_index)); - } else { - self.dyn_content.push((val, tx_index)); - } - self.length += 1; + self.0.push((val, tx_index)) } /// When possible please prefer `get_mut` as it will free From 41483b58a91f04e918a313b48f0bcb8c611fc7a9 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 1 Aug 2019 10:28:04 +0200 Subject: [PATCH 016/129] Missing spaces. --- core/state-machine/src/changes_trie/build.rs | 2 +- core/state-machine/src/overlayed_changes.rs | 32 +++++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 273e21c7ac055..7508eee41cffe 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -225,7 +225,7 @@ mod test { ]); let changes = OverlayedChanges { changes_trie_config: Some(Configuration { digest_interval: 4, digest_levels: 2 }), - changes:OverlayedChangeSet { + changes: OverlayedChangeSet { history: vec![TransactionState::Committed, TransactionState::Pending], children: Default::default(), top: vec![ diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 8bde8a0ff01d0..ff5d106ef32c8 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -150,7 +150,7 @@ impl History { #[cfg(test)] /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator) -> Self { + pub fn from_iter(input: impl IntoIterator) -> Self { let mut history = History::default(); for v in input { history.push(v.0, v.1); @@ -175,8 +175,9 @@ impl History { self.0.push((val, tx_index)) } - /// When possible please prefer `get_mut` as it will free - /// memory. + /// Access to latest pending value (non dropped state in history). + /// When possible please prefer `get_mut` as it can free + /// some memory. fn get(&self, history: &[TransactionState]) -> Option<&V> { // index is never 0, let mut index = self.len(); @@ -272,6 +273,9 @@ impl History { None } + /// Access to latest pending value (non dropped state in history). + /// This method uses `get_mut` and do remove pending + /// This method remove latest dropped value up to the latest valid value. fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { let mut index = self.len(); @@ -299,18 +303,15 @@ impl History { None } + /// Set a value, it uses a state history as parameter. + /// This method uses `get_mut` and do remove pending + /// dropped value. fn set(&mut self, history: &[TransactionState], val: V) { - // TODO EMCH : this is not optimal : can end get_mut as soon as index < state - // needs a variant for get_mut. - match self.get_mut(history) { - Some((v, index)) => { - let state = history.len() - 1; - if index == state { - *v = val; - return; - } - }, - None => (), + if let Some((v, index)) = self.get_mut(history) { + if index == history.len() - 1 { + *v = val; + return; + } } self.push(val, history.len() - 1); } @@ -318,6 +319,9 @@ impl History { } impl History { + + /// Variant of `set` value that update extrinsics. + /// It does remove latest dropped values. fn set_with_extrinsic( &mut self, history: &[TransactionState], val: Option>, From 44f517350f241e8ac1cf863d5b2fcea34fb43c14 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 1 Aug 2019 10:38:02 +0200 Subject: [PATCH 017/129] slice and slice to_vec from ptr to vec. --- core/state-machine/src/ext.rs | 8 ++++---- core/state-machine/src/lib.rs | 2 +- core/state-machine/src/overlayed_changes.rs | 22 ++++++++++----------- core/state-machine/src/testing.rs | 8 ++++---- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 2e3d751eb303a..8666b7f1004e6 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -154,7 +154,7 @@ where self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.changes.top_iter().map(|(k, v)| (k.clone(), v.cloned()))) + .chain(self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec())))) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -273,11 +273,11 @@ where let child_delta_iter = child_storage_keys.map(|storage_key| (storage_key.clone(), self.overlay.changes.child_iter(storage_key) - .map(|(k, v)| (k.clone(), v.cloned())))); + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))))); // compute and memoize - let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.clone(), v.cloned())); + let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter); self.storage_transaction = Some((transaction, root)); @@ -296,7 +296,7 @@ where let storage_key = storage_key.as_ref(); let delta = self.overlay.changes.child_iter(storage_key) - .map(|(k, v)| (k.clone(), v.cloned())); + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); let root = self.backend.child_storage_root(storage_key, delta).0; diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 52dd8cddaad8e..5fe05b3a5e92b 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -1168,7 +1168,7 @@ mod tests { overlay.discard_prospective(); let values: HashMap<_, _> = overlay.changes.top_iter() - .map(|(k, v)| (k.clone(), v.cloned())).collect(); + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))).collect(); assert_eq!( values, map![ diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index ff5d106ef32c8..983f03f206eb2 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -458,43 +458,43 @@ impl OverlayedChangeSet { } /// Iterator over current state of the overlay. - pub fn top_iter_overlay(&self) -> impl Iterator, &OverlayedValue)> { + pub fn top_iter_overlay(&self) -> impl Iterator { self.top.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()).map(|v| (k, v))) + v.get(self.history.as_slice()).map(|v| (k.as_slice(), v))) } /// Iterator over current state of the overlay. - pub fn top_iter(&self) -> impl Iterator, Option<&Vec>)> { - self.top_iter_overlay().map(|(k, v)| (k, v.value.as_ref())) + pub fn top_iter(&self) -> impl Iterator)> { + self.top_iter_overlay().map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) } /// Iterator over current state of the overlay. pub fn child_iter_overlay( &self, storage_key: &[u8], - ) -> impl Iterator, &OverlayedValue)> { + ) -> impl Iterator { self.children.get(storage_key) .into_iter() .flat_map(move |child| child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()).map(|v| (k, v))) + v.get(self.history.as_slice()).map(|v| (k.as_slice(), v))) ) } /// Iterator over current state of the overlay. - pub fn child_iter(&self, storage_key: &[u8]) -> impl Iterator, Option<&Vec>)> { - self.child_iter_overlay(storage_key).map(|(k, v)| (k, v.value.as_ref())) + pub fn child_iter(&self, storage_key: &[u8]) -> impl Iterator)> { + self.child_iter_overlay(storage_key).map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) } /// Iterator over current state of the overlay. pub fn children_iter_overlay( &self, - ) -> impl Iterator, impl Iterator, &OverlayedValue)>)> { + ) -> impl Iterator)> { self.children.iter() - .map(move |(storage_key, child)| (storage_key, child.iter() + .map(move |(storage_key, child)| (storage_key.as_slice(), child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()).map(|v| (k, v))) + v.get(self.history.as_slice()).map(|v| (k.as_slice(), v))) )) } diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 8af2f6d14c3d0..630dc62f31f3d 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -105,11 +105,11 @@ impl TestExternalities { /// Return a new backend with all pending value. pub fn commit_all(&self) -> InMemory { let top = self.overlay.changes.top_iter_overlay() - .map(|(k, v)| (None, k.clone(), v.value.clone())); + .map(|(k, v)| (None, k.to_vec(), v.value.clone())); let children = self.overlay.changes.children_iter_overlay() .flat_map(|(keyspace, map)| map - .map(|(k, v)| (Some(keyspace.clone()), k.clone(), v.value.clone())) + .map(|(k, v)| (Some(keyspace.to_vec()), k.to_vec(), v.value.clone())) .collect::>() ); @@ -223,7 +223,7 @@ impl Externalities for TestExternalities fn storage_root(&mut self) -> H::Out { // compute and memoize - let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.clone(), v.cloned())); + let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); self.backend.storage_root(delta).0 } @@ -233,7 +233,7 @@ impl Externalities for TestExternalities let (root, _, _) = { let delta = self.overlay.changes.child_iter(storage_key) - .map(|(k, v)| (k.clone(), v.cloned())); + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); self.backend.child_storage_root(storage_key, delta) }; self.overlay.set_storage(storage_key.into(), Some(root.clone())); From 00326b2b1f07da23993ba8e57be46b97ea3e44dd Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 1 Aug 2019 16:34:33 +0200 Subject: [PATCH 018/129] Init some fuzzing (only look for panic). --- core/state-machine/fuzz/Cargo.toml | 24 ++++++ .../fuzz/fuzz_targets/fuzz_transactions.rs | 7 ++ core/state-machine/fuzz/src/lib.rs | 80 +++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 core/state-machine/fuzz/Cargo.toml create mode 100644 core/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs create mode 100644 core/state-machine/fuzz/src/lib.rs diff --git a/core/state-machine/fuzz/Cargo.toml b/core/state-machine/fuzz/Cargo.toml new file mode 100644 index 0000000000000..b79a7ff48fe14 --- /dev/null +++ b/core/state-machine/fuzz/Cargo.toml @@ -0,0 +1,24 @@ + +[package] +name = "substrate-state-machine-fuzz" +version = "0.0.1" +authors = [] +publish = false +edition = "2018" + +[package.metadata] +cargo-fuzz = true + +[dependencies.substrate-state-machine] +path = ".." + +[dependencies.libfuzzer-sys] +git = "https://github.com/rust-fuzz/libfuzzer-sys.git" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "fuzz_transactions" +path = "fuzz_targets/fuzz_transactions.rs" diff --git a/core/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs b/core/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs new file mode 100644 index 0000000000000..4de0b91feb5e1 --- /dev/null +++ b/core/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + substrate_state_machine_fuzz::fuzz_transactions(data) +}); diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs new file mode 100644 index 0000000000000..48a5cb747ca89 --- /dev/null +++ b/core/state-machine/fuzz/src/lib.rs @@ -0,0 +1,80 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Substrate state machine fuzzing implementations. +use substrate_state_machine::{ + OverlayedChanges, +}; +/// Size of key, max 255 +const KEY_SPACE: u8 = 20; + +/// Size of key, max 255 +const VALUE_SPACE: u8 = 50; + +pub fn fuzz_transactions(input: &[u8]) { + let mut input_index: usize = 0; + let mut overlayed = OverlayedChanges::default(); + loop { + let action = if let Some(v) = input.get(input_index) { + input_index += 1; + (*v).into() + } else { break }; + + match action { + Actions::CommitProspective => overlayed.commit_prospective(), + Actions::DropProspective => overlayed.discard_prospective(), + Actions::NewTransaction => overlayed.start_transaction(), + Actions::CommitTransaction => overlayed.commit_transaction(), + Actions::DropTransaction => overlayed.discard_transaction(), + Actions::Insert => { + let key = if let Some(v) = input.get(input_index) { + input_index += 1; + v % KEY_SPACE + } else { break }; + let value = if let Some(v) = input.get(input_index) { + input_index += 1; + v % VALUE_SPACE + } else { break }; + overlayed.set_storage(vec![key], Some(vec![value])); + } + } + + } +} + +pub enum Actions { + Insert, + // Delete, same as an insert do not test. + CommitProspective, + DropProspective, + NewTransaction, + CommitTransaction, + DropTransaction, +} + +impl From for Actions { + fn from(v: u8) -> Self { + match (v as usize) * 100 / 255 { + v if v <= 5 => Actions::CommitProspective, + v if v <= 10 => Actions::DropProspective, + v if v <= 20 => Actions::NewTransaction, + v if v <= 30 => Actions::CommitTransaction, + v if v <= 40 => Actions::DropTransaction, + _ => Actions::Insert, + } + } +} + From fc2100d799a5f9f07496a62af7bc8b6f1e574f20 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 1 Aug 2019 19:35:32 +0200 Subject: [PATCH 019/129] Implement simplier inefficient overlay to fuzz compare. Find a bug with fuzzing: fix for commit_transaction (prospective should not break). --- core/state-machine/fuzz/src/lib.rs | 142 +++++++++++++++++++- core/state-machine/src/overlayed_changes.rs | 17 ++- 2 files changed, 146 insertions(+), 13 deletions(-) diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index 48a5cb747ca89..f32e68fc3c997 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -18,6 +18,8 @@ use substrate_state_machine::{ OverlayedChanges, }; +use std::collections::HashMap; + /// Size of key, max 255 const KEY_SPACE: u8 = 20; @@ -27,18 +29,38 @@ const VALUE_SPACE: u8 = 50; pub fn fuzz_transactions(input: &[u8]) { let mut input_index: usize = 0; let mut overlayed = OverlayedChanges::default(); + let mut ref_overlayed = RefOverlayedChanges::default(); + + let mut actions = Vec::new(); + let mut values = Vec::new(); loop { - let action = if let Some(v) = input.get(input_index) { + let action: Actions = if let Some(v) = input.get(input_index) { input_index += 1; (*v).into() } else { break }; + actions.push(action); match action { - Actions::CommitProspective => overlayed.commit_prospective(), - Actions::DropProspective => overlayed.discard_prospective(), - Actions::NewTransaction => overlayed.start_transaction(), - Actions::CommitTransaction => overlayed.commit_transaction(), - Actions::DropTransaction => overlayed.discard_transaction(), + Actions::CommitProspective => { + overlayed.commit_prospective(); + ref_overlayed.commit_prospective(); + }, + Actions::DropProspective => { + overlayed.discard_prospective(); + ref_overlayed.discard_prospective(); + }, + Actions::NewTransaction => { + overlayed.start_transaction(); + ref_overlayed.start_transaction(); + }, + Actions::CommitTransaction => { + overlayed.commit_transaction(); + ref_overlayed.commit_transaction(); + }, + Actions::DropTransaction => { + overlayed.discard_transaction(); + ref_overlayed.discard_transaction(); + }, Actions::Insert => { let key = if let Some(v) = input.get(input_index) { input_index += 1; @@ -48,13 +70,39 @@ pub fn fuzz_transactions(input: &[u8]) { input_index += 1; v % VALUE_SPACE } else { break }; + values.push((key, value)); overlayed.set_storage(vec![key], Some(vec![value])); + ref_overlayed.set_storage(vec![key], Some(vec![value])); } } } + let mut len = 0; + let mut success = true; + for (key, value) in overlayed.top_iter() { + + let ref_value = ref_overlayed.storage(key); + if Some(value) != ref_value { + println!("at {:x?} different values {:x?} {:x?}", key, Some(value), ref_value); + success = false; + } + len += 1; + } + + ref_overlayed.commit_prospective(); + let ref_len = ref_overlayed.committed.len(); + if len != ref_len { + println!("inconsistant length {} {}", len, ref_len); + success = false; + } + if !success { + println!("fuzzing: \n {:x?}", (&actions, &values)); + println!("input: \n {:?}", &input); + } + assert!(success); } +#[derive(Clone, Copy, Debug)] pub enum Actions { Insert, // Delete, same as an insert do not test. @@ -78,3 +126,85 @@ impl From for Actions { } } +/// A simple implementation of overlayed change +/// to use as a comparision. +/// It is partly incomplete (no child trie support, no change trie). +#[derive(Debug, Clone, Default)] +pub struct RefOverlayedChanges { + committed: HashMap, Vec>, + prospective: HashMap, Vec>, + transactions: Vec, Vec>>, +} + +impl RefOverlayedChanges { + pub fn discard_prospective(&mut self) { + self.transactions.clear(); + self.prospective.clear(); + } + + pub fn commit_prospective(&mut self) { + for _ in 0 .. self.transactions.len() { + self.commit_transaction(); + } + self.committed.extend(self.prospective.drain()); + } + + pub fn start_transaction(&mut self) { + self.transactions.push(Default::default()); + } + + pub fn discard_transaction(&mut self) { + if self.transactions.len() == 0 { + // clear prospective on no transaction started. + self.prospective.clear(); + } else { + let _ = self.transactions.pop(); + } + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + match self.transactions.len() { + 0 => (), + 1 => self.prospective.extend( + self.transactions.pop().expect("length just checked").into_iter() + ), + _ => { + let t = self.transactions.pop().expect("length just checked"); + self.transactions.last_mut().expect("length just checked") + .extend(t.into_iter()); + } + } + } + + pub fn set_storage(&mut self, key: Vec, val: Option>) { + if self.transactions.len() > 0 { + self.transactions.last_mut().expect("length just checked") + .insert(key, val.expect("fuzzer do not delete")); + } else { + self.prospective.insert(key, val.expect("fuzzer do not delete")); + } + } + pub fn storage(&self, key: &[u8]) -> Option> { + for t in self.transactions.iter().rev() { + if let Some(v) = t.get(key) { + return Some(Some(v)); + } + } + if let Some(v) = self.prospective.get(key) { + return Some(Some(v)); + } + if let Some(v) = self.committed.get(key) { + return Some(Some(v)); + } + None + } + +} + +#[test] +fn debug_that() { + let input = vec![ + ]; + fuzz_transactions(&input[..]); +} diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 983f03f206eb2..cce2449e8114e 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -71,9 +71,6 @@ pub struct OverlayedValue { /// History of value that are related to a state history (eg field `history` of /// an `OverlayedChangeSet`). /// -/// First field is the current size of the history. -/// Second field are statically allocated values for performance purpose. -/// Third field are dynamically allocated values. /// Values are always paired with a state history index. #[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] @@ -371,6 +368,7 @@ impl History { } } + } impl OverlayedChangeSet { @@ -421,7 +419,6 @@ impl OverlayedChangeSet { /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { let mut i = self.history.len(); - // revert state to previuos pending (or create new pending) while i > 0 { i -= 1; match self.history[i] { @@ -444,14 +441,14 @@ impl OverlayedChangeSet { while i > 0 { i -= 1; match self.history[i] { - TransactionState::Dropped => (), + TransactionState::Prospective + | TransactionState::Dropped => (), TransactionState::Pending => self.history[i] = TransactionState::Prospective, TransactionState::TxPending => { self.history[i] = TransactionState::Prospective; break; }, - TransactionState::Prospective - | TransactionState::Committed => break, + TransactionState::Committed => break, } } self.history.push(TransactionState::Pending); @@ -726,6 +723,12 @@ impl OverlayedChanges { false => None, } } + + /// Iterator over current state of the overlay. + pub fn top_iter(&self) -> impl Iterator)> { + self.changes.top_iter() + } + } #[cfg(test)] From e38fc7b64ecea8dea99baa12703307b7d9d1a6db Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 1 Aug 2019 19:38:40 +0200 Subject: [PATCH 020/129] Update core/state-machine/src/overlayed_changes.rs Co-Authored-By: Sergei Pepyakin --- core/state-machine/src/overlayed_changes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index cce2449e8114e..d90cc8222d5bf 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -74,7 +74,7 @@ pub struct OverlayedValue { /// Values are always paired with a state history index. #[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] -pub(crate) struct History (SmallVec<[(V, usize); ALLOCATED_HISTORY]>); +pub(crate) struct History(SmallVec<[(V, usize); ALLOCATED_HISTORY]>); impl Default for History { fn default() -> Self { From b8564e642e5f0dab572d7cd77f82d94bf30eda65 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 5 Aug 2019 17:42:55 +0200 Subject: [PATCH 021/129] implement overlay garbage collection. --- core/state-machine/fuzz/Cargo.toml | 6 +- .../fuzz_targets/fuzz_transactions_with_gc.rs | 7 + core/state-machine/fuzz/src/lib.rs | 65 ++++++++-- core/state-machine/src/lib.rs | 4 - core/state-machine/src/overlayed_changes.rs | 122 +++++++++++++++++- 5 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_gc.rs diff --git a/core/state-machine/fuzz/Cargo.toml b/core/state-machine/fuzz/Cargo.toml index b79a7ff48fe14..3766de2c146b3 100644 --- a/core/state-machine/fuzz/Cargo.toml +++ b/core/state-machine/fuzz/Cargo.toml @@ -20,5 +20,9 @@ git = "https://github.com/rust-fuzz/libfuzzer-sys.git" members = ["."] [[bin]] -name = "fuzz_transactions" +name = "fuzz_transactions_no_gc" path = "fuzz_targets/fuzz_transactions.rs" + +[[bin]] +name = "fuzz_transactions_with_gc" +path = "fuzz_targets/fuzz_transactions_with_gc.rs" diff --git a/core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_gc.rs b/core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_gc.rs new file mode 100644 index 0000000000000..dda8d261d7b3c --- /dev/null +++ b/core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_gc.rs @@ -0,0 +1,7 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + substrate_state_machine_fuzz::fuzz_transactions_then_gc(data) +}); diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index f32e68fc3c997..878cb7743064f 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -26,7 +26,7 @@ const KEY_SPACE: u8 = 20; /// Size of key, max 255 const VALUE_SPACE: u8 = 50; -pub fn fuzz_transactions(input: &[u8]) { +fn fuzz_transactions_inner(input: &[u8], check_gc: bool) { let mut input_index: usize = 0; let mut overlayed = OverlayedChanges::default(); let mut ref_overlayed = RefOverlayedChanges::default(); @@ -77,31 +77,62 @@ pub fn fuzz_transactions(input: &[u8]) { } } - let mut len = 0; + let mut success = true; - for (key, value) in overlayed.top_iter() { - let ref_value = ref_overlayed.storage(key); - if Some(value) != ref_value { - println!("at {:x?} different values {:x?} {:x?}", key, Some(value), ref_value); + let (check_value, len) = check_values(&overlayed, &ref_overlayed); + success &= check_value; + + if check_gc { + let reference_size = ref_overlayed.total_length(); + overlayed.gc(true); + let size = overlayed.top_count_keyvalue_pair(); + if reference_size != size { + println!("inconsistent gc {} {}", size, reference_size); success = false; } - len += 1; + let (check_value, len_gc) = check_values(&overlayed, &ref_overlayed); + success &= check_value; + success &= len_gc == len; } - ref_overlayed.commit_prospective(); let ref_len = ref_overlayed.committed.len(); if len != ref_len { - println!("inconsistant length {} {}", len, ref_len); + println!("inconsistent length {} {}", len, ref_len); success = false; } if !success { println!("fuzzing: \n {:x?}", (&actions, &values)); println!("input: \n {:?}", &input); } + + assert!(success); } +fn check_values(overlayed: &OverlayedChanges, ref_overlayed: &RefOverlayedChanges) -> (bool, usize) { + let mut len = 0; + let mut success = true; + for (key, value) in overlayed.top_iter() { + + let ref_value = ref_overlayed.storage(key); + if Some(value) != ref_value { + println!("at {:x?} different values {:x?} {:x?}", key, Some(value), ref_value); + success = false; + } + len += 1; + } + (success, len) +} + +pub fn fuzz_transactions(input: &[u8]) { + fuzz_transactions_inner(input, false); +} + +pub fn fuzz_transactions_then_gc(input: &[u8]) { + fuzz_transactions_inner(input, true); +} + #[derive(Clone, Copy, Debug)] pub enum Actions { Insert, @@ -185,6 +216,7 @@ impl RefOverlayedChanges { self.prospective.insert(key, val.expect("fuzzer do not delete")); } } + pub fn storage(&self, key: &[u8]) -> Option> { for t in self.transactions.iter().rev() { if let Some(v) = t.get(key) { @@ -200,11 +232,24 @@ impl RefOverlayedChanges { None } + pub fn total_length(&self) -> usize { + let tr_len: usize = self.transactions.iter() + .map(|l| l.len()).sum(); + self.committed.len() + self.prospective.len() + tr_len + } } #[test] fn debug_that() { +//50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255 + +//238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61 + +//50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67 + +//0x6e,0xff,0xf7,0x0,0x6e,0xff,0xff,0x2d,0xff,0xff,0xff,0xe + let input = vec![ ]; - fuzz_transactions(&input[..]); + fuzz_transactions_inner(&input[..], true); } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 5fe05b3a5e92b..3e6e5a446572c 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -639,10 +639,6 @@ impl<'a, H, N, B, T, O, Exec> StateMachine<'a, H, N, B, T, O, Exec> where } } - // TODO EMCH in all exec the prospective is replace by the full state (commited included) - // -> TODO bench over using a check point approach (expect ctommit prospective to not be call), - // or agains a gc checkpoint approach. - /// Execute a call using the given state backend, overlayed changes, and call executor. /// Produces a state-backend-specific "transaction" which can be used to apply the changes /// to the backing store, such as the disk. diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index d90cc8222d5bf..30de8ba7b3d0a 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -287,9 +287,14 @@ impl History { index -= 1; let history_index = self[index].1; match history[history_index] { + TransactionState::Committed => { + // here we could gc all preceding values but that is additional cost + // and get_mut should stop at pending following committed. + let tx = self[index].1; + return Some((&mut self[index].0, tx)) + }, TransactionState::Pending | TransactionState::TxPending - | TransactionState::Committed | TransactionState::Prospective => { let tx = self[index].1; return Some((&mut self[index].0, tx)) @@ -313,6 +318,82 @@ impl History { self.push(val, history.len() - 1); } + /// Garbage collect a history, act as a `get_mut` with additional cost. + /// If `eager` is true, all dropped value are removed even if it means shifting + /// array byte. Otherwhise we mainly ensure collection up to last Commit state + /// (truncate left size). + fn gc( + &mut self, + history: &[TransactionState], + tx_index: Option<&[usize]>, + ) -> Option<(&mut V, usize)> { + if let Some(tx_index) = tx_index { + let mut tx_index = &tx_index[..]; + let mut index = self.len(); + if index == 0 { + return None; + } + let mut bellow_value = usize::max_value(); + let mut result: Option = None; + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Committed => { + for _ in 0..index { + let _ = self.0.remove(0); + } + result = Some(result.map(|i| i - index).unwrap_or(0)); + index = 0; + }, + TransactionState::Pending + | TransactionState::Prospective => { + if history_index >= bellow_value { + let _ = self.0.remove(index); + result.as_mut().map(|i| *i = *i - 1); + } else { + if result.is_none() { + result = Some(index); + } + while bellow_value > history_index { + // bellow_value = pop + let split = tx_index.split_last() + .map(|(v, sl)| (*v, sl)) + .unwrap_or((0, &[])); + bellow_value = split.0; + tx_index = split.1; + } + } + }, + TransactionState::TxPending => { + if history_index >= bellow_value { + let _ = self.0.remove(index); + result.as_mut().map(|i| *i = *i - 1); + } else { + if result.is_none() { + result = Some(index); + } + } + bellow_value = usize::max_value(); + }, + TransactionState::Dropped => { + let _ = self.0.remove(index); + }, + } + } + if let Some(index) = result { + let tx = self[index].1.clone(); + Some((&mut self[index].0, tx)) + } else { None } + + } else { + return self.get_mut(history); + } + } } impl History { @@ -372,6 +453,29 @@ impl History { } impl OverlayedChangeSet { + /// Garbage collect. + fn gc(&mut self, eager: bool) { + let eager = if eager { + let mut tx_ixs = Vec::new(); + for (i, h) in self.history.iter().enumerate() { + if &TransactionState::TxPending == h { + tx_ixs.push(i); + } + } + Some(tx_ixs) + } else { + None + }; + let history = self.history.as_slice(); + // retain does change values + self.top.retain(|_, h| h.gc(history, eager.as_ref().map(|t| t.as_slice())).is_some()); + self.children.retain(|_, m| { + m.retain(|_, h| h.gc(history, eager.as_ref().map(|t| t.as_slice())).is_some()); + m.len() > 0 + }); + } + + /// Whether the change set is empty. pub fn is_empty(&self) -> bool { self.top.is_empty() && self.children.is_empty() @@ -729,6 +833,22 @@ impl OverlayedChanges { self.changes.top_iter() } + /// Count (slow) the number of key value, history included. + /// Only for debugging or testing usage. + pub fn top_count_keyvalue_pair(&self) -> usize { + let mut result = 0; + for (_, v) in self.changes.top.iter() { + result += v.0.len() + } + result + } + + /// costy garbage collection of unneeded memory from + /// key values. Eager set to true will remove more + /// key value but allows more costy memory changes. + pub fn gc(&mut self, eager: bool) { + self.changes.gc(eager); + } } #[cfg(test)] From 21aa65d96cf41352c4271ccedf284c1098fff709 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 5 Aug 2019 18:41:48 +0200 Subject: [PATCH 022/129] Trigger gc based on constant rule. --- core/state-machine/src/changes_trie/build.rs | 1 + core/state-machine/src/ext.rs | 1 + core/state-machine/src/overlayed_changes.rs | 59 +++++++++++++++++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 7508eee41cffe..ce2fd3a212dd6 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -259,6 +259,7 @@ mod test { ])), ].into_iter().collect(), }, + operation_from_last_gc: 0, }; (backend, storage, changes) diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 8666b7f1004e6..93cbb1dcd6d82 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -380,6 +380,7 @@ mod tests { ])), ].into_iter().collect(), }, + operation_from_last_gc: 0, } } diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 30de8ba7b3d0a..26081dec54e49 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -44,6 +44,25 @@ pub(crate) enum TransactionState { Dropped, } + +/// Treshold of operation before running a garbage colletion +/// on a transaction operation. +/// Should be same as `TRIGGER_COMMIT_GC` or higher +/// (we most likely do not want lower as transaction are +/// possibly more frequent than commit). +const TRIGGER_TRANSACTION_GC: usize = 10_000; + +/// Treshold of operation before running a garbage colletion +/// on a commit operation. +/// We may want a lower value than for a transaction, even +/// a 1 if we want to do it between every operation. +const TRIGGER_COMMIT_GC: usize = 1_000; + +/// Used to count big content as multiple operation. +/// This is a number of octet. +/// Set to 0 to ignore. +const ADD_CONTENT_SIZE_UNIT: usize = 64; + /// The overlayed changes to state to be queried on top of the backend. /// /// A transaction shares all prospective changes within an inner overlay @@ -55,6 +74,10 @@ pub struct OverlayedChanges { /// Changes trie configuration. None by default, but could be installed by the /// runtime if it supports change tries. pub(crate) changes_trie_config: Option, + /// Counter of number of operation between garbage collection. + /// Add or delete cost one, additional cost per size by counting a fix size + /// as a unit. + pub(crate) operation_from_last_gc: usize, } /// The storage value, used inside OverlayedChanges. @@ -683,10 +706,18 @@ impl OverlayedChanges { None } + fn add_cost_op(&mut self, val: &Option>) { + let content_cost = if ADD_CONTENT_SIZE_UNIT > 0 { + val.as_ref().map(|s| s.len() / ADD_CONTENT_SIZE_UNIT).unwrap_or(0) + } else { 0 }; + self.operation_from_last_gc += 1 + content_cost; + } + /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. pub fn set_storage(&mut self, key: Vec, val: Option>) { + self.add_cost_op(&val); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); entry.set_with_extrinsic(self.changes.history.as_slice(), val, extrinsic_index); @@ -696,6 +727,7 @@ impl OverlayedChanges { /// /// `None` can be used to delete a value specified by the given key. pub(crate) fn set_child_storage(&mut self, storage_key: Vec, key: Vec, val: Option>) { + self.add_cost_op(&val); let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); @@ -713,6 +745,7 @@ impl OverlayedChanges { let history = self.changes.history.as_slice(); let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); + self.operation_from_last_gc += map_entry.len(); map_entry.values_mut().for_each(|e| e.set_with_extrinsic(history, None, extrinsic_index)); } @@ -725,32 +758,52 @@ impl OverlayedChanges { pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); + let mut nb_remove = 0; for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { + nb_remove += 1; entry.set_with_extrinsic(self.changes.history.as_slice(), None, extrinsic_index); } } + + self.operation_from_last_gc += nb_remove; } /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.changes.discard_prospective(); + if self.operation_from_last_gc > TRIGGER_COMMIT_GC { + self.operation_from_last_gc = 0; + self.gc(true); + } } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.changes.commit_prospective(); + if self.operation_from_last_gc > TRIGGER_COMMIT_GC { + self.operation_from_last_gc = 0; + self.gc(true); + } } /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.changes.start_transaction(); + if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { + self.operation_from_last_gc = 0; + self.gc(true); + } } /// Discard a transactional layer. /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { self.changes.discard_transaction(); + if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { + self.operation_from_last_gc = 0; + self.gc(true); + } } /// Commit a transactional layer. @@ -803,7 +856,11 @@ impl OverlayedChanges { changes_trie_config: Option, ) -> Self { let changes = OverlayedChangeSet::default(); - let mut result = OverlayedChanges { changes, changes_trie_config }; + let mut result = OverlayedChanges { + changes, + changes_trie_config, + operation_from_last_gc: 0, + }; committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); result.changes.commit_prospective(); prospective.into_iter().for_each(|(k, v)| result.set_storage(k, v)); From c8938d3cf003a51d55ef3db54f5f20171db2e562 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 5 Aug 2019 20:10:29 +0200 Subject: [PATCH 023/129] Managing a configuration for gc of overlay. This commit will be revert (static value makes more sense for the time being). --- core/client/src/backend.rs | 4 ++ core/client/src/call_executor.rs | 1 + core/state-machine/src/changes_trie/build.rs | 1 + core/state-machine/src/ext.rs | 1 + core/state-machine/src/lib.rs | 2 +- core/state-machine/src/overlayed_changes.rs | 64 ++++++++++++++------ 6 files changed, 54 insertions(+), 19 deletions(-) diff --git a/core/client/src/backend.rs b/core/client/src/backend.rs index 24b48c9b8618c..213b278e58685 100644 --- a/core/client/src/backend.rs +++ b/core/client/src/backend.rs @@ -23,6 +23,7 @@ use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenSto use sr_primitives::traits::{Block as BlockT, NumberFor}; use state_machine::backend::Backend as StateBackend; use state_machine::ChangesTrieStorage as StateChangesTrieStorage; +use state_machine::OverlayedSettings; use consensus::well_known_cache_keys; use hash_db::Hasher; use trie::MemoryDB; @@ -196,6 +197,9 @@ pub trait Backend: AuxStore + Send + Sync where /// something that the import of a block would interfere with, e.g. importing /// a new block or calculating the best head. fn get_import_lock(&self) -> &Mutex<()>; + + /// Settings to use with overlay. + fn overlay_settings(&self) -> OverlayedSettings; } /// Offchain workers local storage. diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index 1b41cc3aa38eb..712fb25e79de0 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -188,6 +188,7 @@ where side_effects_handler: Option<&mut O>, ) -> error::Result> { let mut changes = OverlayedChanges::default(); + changes.change_settings(self.backend.overlay_settings()); let state = self.backend.state_at(*id)?; let return_data = state_machine::new( &state, diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index ce2fd3a212dd6..cca16f3c2f3d3 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -259,6 +259,7 @@ mod test { ])), ].into_iter().collect(), }, + settings: Default::default(), operation_from_last_gc: 0, }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 93cbb1dcd6d82..ae2b30c37295f 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -380,6 +380,7 @@ mod tests { ])), ].into_iter().collect(), }, + settings: Default::default(), operation_from_last_gc: 0, } } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 3e6e5a446572c..de592ecc77fd4 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -52,7 +52,7 @@ pub use changes_trie::{ prune as prune_changes_tries, oldest_non_pruned_trie as oldest_non_pruned_changes_trie }; -pub use overlayed_changes::OverlayedChanges; +pub use overlayed_changes::{OverlayedChanges, OverlayedSettings}; pub use proving_backend::{ create_proof_check_backend, create_proof_check_backend_storage, Recorder as ProofRecorder, ProvingBackend, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 26081dec54e49..381ab2f54e225 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -45,24 +45,42 @@ pub(crate) enum TransactionState { } -/// Treshold of operation before running a garbage colletion -/// on a transaction operation. -/// Should be same as `TRIGGER_COMMIT_GC` or higher -/// (we most likely do not want lower as transaction are -/// possibly more frequent than commit). const TRIGGER_TRANSACTION_GC: usize = 10_000; -/// Treshold of operation before running a garbage colletion -/// on a commit operation. -/// We may want a lower value than for a transaction, even -/// a 1 if we want to do it between every operation. const TRIGGER_COMMIT_GC: usize = 1_000; -/// Used to count big content as multiple operation. -/// This is a number of octet. -/// Set to 0 to ignore. const ADD_CONTENT_SIZE_UNIT: usize = 64; +#[derive(Debug, Clone)] +/// Settings regarding overlayed changes internal (mainly gc). +pub struct OverlayedSettings { + /// Treshold of operation before running a garbage colletion + /// on a transaction operation. + /// Should be same as `TRIGGER_COMMIT_GC` or higher + /// (we most likely do not want lower as transaction are + /// possibly more frequent than commit). + pub trigger_transaction_gc: usize, + /// Treshold of operation before running a garbage colletion + /// on a commit operation. + /// We may want a lower value than for a transaction, even + /// a 1 if we want to do it between every operation. + pub trigger_commit_gc: usize, + /// Used to count big content as multiple operation. + /// This is a number of octet. + /// Set to 0 to ignore. + pub add_content_size_unit: usize, +} + +impl Default for OverlayedSettings { + fn default() -> Self { + OverlayedSettings { + trigger_transaction_gc: TRIGGER_TRANSACTION_GC, + trigger_commit_gc: TRIGGER_COMMIT_GC, + add_content_size_unit: ADD_CONTENT_SIZE_UNIT, + } + } +} + /// The overlayed changes to state to be queried on top of the backend. /// /// A transaction shares all prospective changes within an inner overlay @@ -74,6 +92,8 @@ pub struct OverlayedChanges { /// Changes trie configuration. None by default, but could be installed by the /// runtime if it supports change tries. pub(crate) changes_trie_config: Option, + /// Settings regarding overlayed changes internal (mainly gc). + pub(crate) settings: OverlayedSettings, /// Counter of number of operation between garbage collection. /// Add or delete cost one, additional cost per size by counting a fix size /// as a unit. @@ -652,6 +672,13 @@ impl OverlayedChangeSet { } impl OverlayedChanges { + + /// Change overlayed change settings. + pub fn change_settings(&mut self, settings: OverlayedSettings) { + self.settings = settings; + } + + /// Whether the overlayed changes are empty. pub fn is_empty(&self) -> bool { self.changes.is_empty() @@ -707,8 +734,8 @@ impl OverlayedChanges { } fn add_cost_op(&mut self, val: &Option>) { - let content_cost = if ADD_CONTENT_SIZE_UNIT > 0 { - val.as_ref().map(|s| s.len() / ADD_CONTENT_SIZE_UNIT).unwrap_or(0) + let content_cost = if self.settings.add_content_size_unit > 0 { + val.as_ref().map(|s| s.len() / self.settings.add_content_size_unit).unwrap_or(0) } else { 0 }; self.operation_from_last_gc += 1 + content_cost; } @@ -772,7 +799,7 @@ impl OverlayedChanges { /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.changes.discard_prospective(); - if self.operation_from_last_gc > TRIGGER_COMMIT_GC { + if self.operation_from_last_gc > self.settings.trigger_commit_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -781,7 +808,7 @@ impl OverlayedChanges { /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.changes.commit_prospective(); - if self.operation_from_last_gc > TRIGGER_COMMIT_GC { + if self.operation_from_last_gc > self.settings.trigger_commit_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -790,7 +817,7 @@ impl OverlayedChanges { /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.changes.start_transaction(); - if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { + if self.operation_from_last_gc > self.settings.trigger_transaction_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -800,7 +827,7 @@ impl OverlayedChanges { /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { self.changes.discard_transaction(); - if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { + if self.operation_from_last_gc > self.settings.trigger_transaction_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -859,6 +886,7 @@ impl OverlayedChanges { let mut result = OverlayedChanges { changes, changes_trie_config, + settings: Default::default(), operation_from_last_gc: 0, }; committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); From d1d947bd4fc8b064b4b2dad49efc24cd16ff9f47 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 5 Aug 2019 20:11:55 +0200 Subject: [PATCH 024/129] Revert "Managing a configuration for gc of overlay. This commit will be revert" This reverts commit c8938d3cf003a51d55ef3db54f5f20171db2e562. --- core/client/src/backend.rs | 4 -- core/client/src/call_executor.rs | 1 - core/state-machine/src/changes_trie/build.rs | 1 - core/state-machine/src/ext.rs | 1 - core/state-machine/src/lib.rs | 2 +- core/state-machine/src/overlayed_changes.rs | 64 ++++++-------------- 6 files changed, 19 insertions(+), 54 deletions(-) diff --git a/core/client/src/backend.rs b/core/client/src/backend.rs index 213b278e58685..24b48c9b8618c 100644 --- a/core/client/src/backend.rs +++ b/core/client/src/backend.rs @@ -23,7 +23,6 @@ use sr_primitives::{generic::BlockId, Justification, StorageOverlay, ChildrenSto use sr_primitives::traits::{Block as BlockT, NumberFor}; use state_machine::backend::Backend as StateBackend; use state_machine::ChangesTrieStorage as StateChangesTrieStorage; -use state_machine::OverlayedSettings; use consensus::well_known_cache_keys; use hash_db::Hasher; use trie::MemoryDB; @@ -197,9 +196,6 @@ pub trait Backend: AuxStore + Send + Sync where /// something that the import of a block would interfere with, e.g. importing /// a new block or calculating the best head. fn get_import_lock(&self) -> &Mutex<()>; - - /// Settings to use with overlay. - fn overlay_settings(&self) -> OverlayedSettings; } /// Offchain workers local storage. diff --git a/core/client/src/call_executor.rs b/core/client/src/call_executor.rs index 712fb25e79de0..1b41cc3aa38eb 100644 --- a/core/client/src/call_executor.rs +++ b/core/client/src/call_executor.rs @@ -188,7 +188,6 @@ where side_effects_handler: Option<&mut O>, ) -> error::Result> { let mut changes = OverlayedChanges::default(); - changes.change_settings(self.backend.overlay_settings()); let state = self.backend.state_at(*id)?; let return_data = state_machine::new( &state, diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index cca16f3c2f3d3..ce2fd3a212dd6 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -259,7 +259,6 @@ mod test { ])), ].into_iter().collect(), }, - settings: Default::default(), operation_from_last_gc: 0, }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index ae2b30c37295f..93cbb1dcd6d82 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -380,7 +380,6 @@ mod tests { ])), ].into_iter().collect(), }, - settings: Default::default(), operation_from_last_gc: 0, } } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index de592ecc77fd4..3e6e5a446572c 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -52,7 +52,7 @@ pub use changes_trie::{ prune as prune_changes_tries, oldest_non_pruned_trie as oldest_non_pruned_changes_trie }; -pub use overlayed_changes::{OverlayedChanges, OverlayedSettings}; +pub use overlayed_changes::OverlayedChanges; pub use proving_backend::{ create_proof_check_backend, create_proof_check_backend_storage, Recorder as ProofRecorder, ProvingBackend, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 381ab2f54e225..26081dec54e49 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -45,42 +45,24 @@ pub(crate) enum TransactionState { } +/// Treshold of operation before running a garbage colletion +/// on a transaction operation. +/// Should be same as `TRIGGER_COMMIT_GC` or higher +/// (we most likely do not want lower as transaction are +/// possibly more frequent than commit). const TRIGGER_TRANSACTION_GC: usize = 10_000; +/// Treshold of operation before running a garbage colletion +/// on a commit operation. +/// We may want a lower value than for a transaction, even +/// a 1 if we want to do it between every operation. const TRIGGER_COMMIT_GC: usize = 1_000; +/// Used to count big content as multiple operation. +/// This is a number of octet. +/// Set to 0 to ignore. const ADD_CONTENT_SIZE_UNIT: usize = 64; -#[derive(Debug, Clone)] -/// Settings regarding overlayed changes internal (mainly gc). -pub struct OverlayedSettings { - /// Treshold of operation before running a garbage colletion - /// on a transaction operation. - /// Should be same as `TRIGGER_COMMIT_GC` or higher - /// (we most likely do not want lower as transaction are - /// possibly more frequent than commit). - pub trigger_transaction_gc: usize, - /// Treshold of operation before running a garbage colletion - /// on a commit operation. - /// We may want a lower value than for a transaction, even - /// a 1 if we want to do it between every operation. - pub trigger_commit_gc: usize, - /// Used to count big content as multiple operation. - /// This is a number of octet. - /// Set to 0 to ignore. - pub add_content_size_unit: usize, -} - -impl Default for OverlayedSettings { - fn default() -> Self { - OverlayedSettings { - trigger_transaction_gc: TRIGGER_TRANSACTION_GC, - trigger_commit_gc: TRIGGER_COMMIT_GC, - add_content_size_unit: ADD_CONTENT_SIZE_UNIT, - } - } -} - /// The overlayed changes to state to be queried on top of the backend. /// /// A transaction shares all prospective changes within an inner overlay @@ -92,8 +74,6 @@ pub struct OverlayedChanges { /// Changes trie configuration. None by default, but could be installed by the /// runtime if it supports change tries. pub(crate) changes_trie_config: Option, - /// Settings regarding overlayed changes internal (mainly gc). - pub(crate) settings: OverlayedSettings, /// Counter of number of operation between garbage collection. /// Add or delete cost one, additional cost per size by counting a fix size /// as a unit. @@ -672,13 +652,6 @@ impl OverlayedChangeSet { } impl OverlayedChanges { - - /// Change overlayed change settings. - pub fn change_settings(&mut self, settings: OverlayedSettings) { - self.settings = settings; - } - - /// Whether the overlayed changes are empty. pub fn is_empty(&self) -> bool { self.changes.is_empty() @@ -734,8 +707,8 @@ impl OverlayedChanges { } fn add_cost_op(&mut self, val: &Option>) { - let content_cost = if self.settings.add_content_size_unit > 0 { - val.as_ref().map(|s| s.len() / self.settings.add_content_size_unit).unwrap_or(0) + let content_cost = if ADD_CONTENT_SIZE_UNIT > 0 { + val.as_ref().map(|s| s.len() / ADD_CONTENT_SIZE_UNIT).unwrap_or(0) } else { 0 }; self.operation_from_last_gc += 1 + content_cost; } @@ -799,7 +772,7 @@ impl OverlayedChanges { /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.changes.discard_prospective(); - if self.operation_from_last_gc > self.settings.trigger_commit_gc { + if self.operation_from_last_gc > TRIGGER_COMMIT_GC { self.operation_from_last_gc = 0; self.gc(true); } @@ -808,7 +781,7 @@ impl OverlayedChanges { /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.changes.commit_prospective(); - if self.operation_from_last_gc > self.settings.trigger_commit_gc { + if self.operation_from_last_gc > TRIGGER_COMMIT_GC { self.operation_from_last_gc = 0; self.gc(true); } @@ -817,7 +790,7 @@ impl OverlayedChanges { /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.changes.start_transaction(); - if self.operation_from_last_gc > self.settings.trigger_transaction_gc { + if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { self.operation_from_last_gc = 0; self.gc(true); } @@ -827,7 +800,7 @@ impl OverlayedChanges { /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { self.changes.discard_transaction(); - if self.operation_from_last_gc > self.settings.trigger_transaction_gc { + if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { self.operation_from_last_gc = 0; self.gc(true); } @@ -886,7 +859,6 @@ impl OverlayedChanges { let mut result = OverlayedChanges { changes, changes_trie_config, - settings: Default::default(), operation_from_last_gc: 0, }; committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); From ee52917da8ea0dd94f79018fd1efc39e05e44c08 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 15 Aug 2019 12:31:58 +0200 Subject: [PATCH 025/129] Direct iterator on children, not that the fact that it is owned does not really seems required. --- core/state-machine/src/ext.rs | 9 +------ core/state-machine/src/overlayed_changes.rs | 26 ++++++++++++++++++++- core/state-machine/src/testing.rs | 6 +---- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 7caecc679f754..d9ff52ae1c52e 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -298,14 +298,7 @@ where return root.clone(); } - // TODO EMCH this buffer can be remove : create a children_iter function!! - // -> replace for all child_iter of this kind. - let child_storage_keys = self.overlay.changes.children.keys(); - - let child_delta_iter = child_storage_keys.map(|storage_key| - (storage_key.clone(), self.overlay.changes.child_iter(storage_key) - .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))))); - + let child_delta_iter = self.overlay.changes.owned_children_iter(); // compute and memoize let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index e5e360a201a6c..9f7ce190b3b79 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -616,12 +616,36 @@ impl OverlayedChangeSet { &self, ) -> impl Iterator)> { self.children.iter() - .map(move |(storage_key, child)| (storage_key.as_slice(), child.iter() + .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() .filter_map(move |(k, v)| v.get(self.history.as_slice()).map(|v| (k.as_slice(), v))) )) } + /// Iterator over current state of the overlay. + pub fn children_iter( + &self, + ) -> impl Iterator)>)> { + self.children.iter() + .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() + .filter_map(move |(k, v)| + v.get(self.history.as_slice()) + .map(|v| (k.as_slice(), v.value.as_ref().map(|v| v.as_slice())))) + )) + } + + /// Iterator over current state of the overlay. + pub fn owned_children_iter<'a>( + &'a self, + ) -> impl Iterator, impl Iterator, Option>)> + 'a)> + 'a { + self.children.iter() + .map(move |(keyspace, child)| (keyspace.to_vec(), child.iter() + .filter_map(move |(k, v)| + v.get(self.history.as_slice()) + .map(|v| (k.to_vec(), v.value.as_ref().map(|v| v.to_vec())))) + )) + } + /// Test only method to access current prospective changes. /// It is here to keep old test compatibility and should be /// avoid for new tests. diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index c5f690e3a35ea..2a898d6b23546 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -221,11 +221,7 @@ impl Externalities for TestExternalities fn storage_root(&mut self) -> H::Out { - let child_storage_keys = self.overlay.changes.children.keys(); - - let child_delta_iter = child_storage_keys.map(|storage_key| - (storage_key.clone(), self.overlay.changes.child_iter(storage_key) - .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))))); + let child_delta_iter = self.overlay.changes.owned_children_iter(); // compute and memoize let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); From af4cceb0b8d1a99f1396768ad079d933209593c3 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 26 Aug 2019 08:51:00 +0200 Subject: [PATCH 026/129] warning fix --- core/state-machine/src/overlayed_changes.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index a7d122a606f78..f727aca687b44 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -877,7 +877,6 @@ impl OverlayedChanges { #[cfg(test)] pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) { use codec::Encode; - let value = extrinsic_index.encode(); self.set_storage(EXTRINSIC_INDEX.to_vec(), Some(extrinsic_index.encode())); } From 4e68123ed8eed86e91902f896c25a84b917993c6 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 26 Aug 2019 15:42:16 +0200 Subject: [PATCH 027/129] Extracting transaction history and states plus logics to its own no_std compatible crate: history-driven-data. --- Cargo.lock | 11 +- Cargo.toml | 1 + core/sr-api-macros/tests/runtime_calls.rs | 8 + core/state-machine/Cargo.toml | 4 +- core/state-machine/fuzz/Cargo.toml | 1 + core/state-machine/src/changes_trie/build.rs | 6 +- core/state-machine/src/ext.rs | 6 +- core/state-machine/src/overlayed_changes.rs | 521 +++---------------- core/test-runtime/Cargo.toml | 2 + core/test-runtime/src/lib.rs | 38 ++ core/utils/history-driven-data/Cargo.toml | 20 + core/utils/history-driven-data/README.md | 20 + core/utils/history-driven-data/src/lib.rs | 68 +++ core/utils/history-driven-data/src/linear.rs | 436 ++++++++++++++++ 14 files changed, 693 insertions(+), 449 deletions(-) create mode 100644 core/utils/history-driven-data/Cargo.toml create mode 100644 core/utils/history-driven-data/README.md create mode 100644 core/utils/history-driven-data/src/lib.rs create mode 100644 core/utils/history-driven-data/src/linear.rs diff --git a/Cargo.lock b/Cargo.lock index e646ffb2f80b0..f9c3896063e32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1212,6 +1212,14 @@ dependencies = [ "proc-macro-hack 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "history-driven-data" +version = "2.0.0" +dependencies = [ + "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "sr-std 2.0.0", +] + [[package]] name = "hmac" version = "0.4.2" @@ -5122,12 +5130,12 @@ dependencies = [ "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "history-driven-data 2.0.0", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-panic-handler 2.0.0", "substrate-primitives 2.0.0", "substrate-trie 2.0.0", @@ -5178,6 +5186,7 @@ name = "substrate-test-runtime" version = "2.0.0" dependencies = [ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "history-driven-data 2.0.0", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 048bfb7629060..91e26823fecaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ members = [ "core/transaction-pool/graph", "core/trie", "core/utils/fork-tree", + "core/utils/history-driven-data", "core/utils/wasm-builder", "core/utils/wasm-builder-runner", "srml/support", diff --git a/core/sr-api-macros/tests/runtime_calls.rs b/core/sr-api-macros/tests/runtime_calls.rs index b779348e27542..6f5dedbf886dd 100644 --- a/core/sr-api-macros/tests/runtime_calls.rs +++ b/core/sr-api-macros/tests/runtime_calls.rs @@ -127,6 +127,14 @@ fn use_trie_function() { assert_eq!(runtime_api.use_trie(&block_id).unwrap(), 2); } +#[test] +fn use_history_data() { + let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); + let runtime_api = client.runtime_api(); + let block_id = BlockId::Number(client.info().chain.best_number); + assert_eq!(runtime_api.use_history_data(&block_id).unwrap(), 0); +} + #[test] fn test_transactions() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index 0aeb073f8b751..cfeb687f6beb8 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -16,12 +16,13 @@ primitives = { package = "substrate-primitives", path = "../primitives" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } codec = { package = "parity-scale-codec", version = "1.0.0" } num-traits = "0.2" -smallvec = "0.6" +history-driven-data = { path = "../utils/history-driven-data", features = ["std"] } [dev-dependencies] hex-literal = "0.2.0" criterion = "0.2" rand = "0.6" +history-driven-data = { path = "../utils/history-driven-data", features = ["std", "test"] } [[bench]] name = "bench" @@ -29,3 +30,4 @@ harness = false [features] default = [] +test = [] diff --git a/core/state-machine/fuzz/Cargo.toml b/core/state-machine/fuzz/Cargo.toml index 3766de2c146b3..90afcdf558cd1 100644 --- a/core/state-machine/fuzz/Cargo.toml +++ b/core/state-machine/fuzz/Cargo.toml @@ -11,6 +11,7 @@ cargo-fuzz = true [dependencies.substrate-state-machine] path = ".." +features = ["test"] [dependencies.libfuzzer-sys] git = "https://github.com/rust-fuzz/libfuzzer-sys.git" diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 0bf4d72267332..85264e38c7ed1 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -177,7 +177,9 @@ mod test { use primitives::storage::well_known_keys::EXTRINSIC_INDEX; use crate::backend::InMemory; use crate::changes_trie::storage::InMemoryStorage; - use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet, History, TransactionState}; + use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; + use history_driven_data::linear::{History, States}; + use history_driven_data::State as TransactionState; use super::*; fn prepare_for_build() -> (InMemory, InMemoryStorage, OverlayedChanges) { @@ -226,7 +228,7 @@ mod test { let changes = OverlayedChanges { changes_trie_config: Some(Configuration { digest_interval: 4, digest_levels: 2 }), changes: OverlayedChangeSet { - history: vec![TransactionState::Committed, TransactionState::Pending], + history: States::test_vector(vec![TransactionState::Committed, TransactionState::Pending]), children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index d9ff52ae1c52e..8b7317b3d8f43 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -375,7 +375,9 @@ mod tests { use crate::backend::InMemory; use crate::changes_trie::{Configuration as ChangesTrieConfiguration, InMemoryStorage as InMemoryChangesTrieStorage}; - use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet, History, TransactionState}; + use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; + use history_driven_data::linear::History; + use history_driven_data::State as TransactionState; use super::*; type TestBackend = InMemory; @@ -390,7 +392,7 @@ mod tests { digest_levels: 0, }), changes: OverlayedChangeSet { - history: vec![TransactionState::Pending], + history: Default::default(), children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index f727aca687b44..5273e742e8784 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -21,44 +21,24 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use smallvec::SmallVec; - -#[derive(Debug, Clone, PartialEq)] -/// State of a transaction, the states are stored -/// in a linear indexed history. -pub(crate) enum TransactionState { - /// Overlay is under change and can still be dropped. - Pending, - /// Overlay is under change and can still be dropped. - /// This also mark the start of a transaction. - TxPending, - /// Information is committed, but can still be dropped - /// from `discard_prospective` or `discard_transaction` - /// of a parent transaction. - Prospective, - /// Committed is information that cannot be dropped. - Committed, - /// Transaction or prospective has been dropped. - /// Information pointing to this indexed historic state should - /// not be returned and can be removed. - Dropped, -} - +use history_driven_data::linear::{States, History}; +use history_driven_data::State as TransactionState; -/// Treshold of operation before running a garbage colletion -/// on a transaction operation. +/// Treshold in number of operation before running a garbage colletion. +/// /// Should be same as `TRIGGER_COMMIT_GC` or higher /// (we most likely do not want lower as transaction are /// possibly more frequent than commit). const TRIGGER_TRANSACTION_GC: usize = 100_000; -/// Treshold of operation before running a garbage colletion +/// Treshold in number of operation before running a garbage colletion /// on a commit operation. +/// /// We may want a lower value than for a transaction, even /// a 1 if we want to do it between every operation. const TRIGGER_COMMIT_GC: usize = 10_000; -/// Used to count big content as multiple operation. +/// Used to count big content as multiple operations. /// This is a number of octet. /// Set to 0 to ignore. const ADD_CONTENT_SIZE_UNIT: usize = 64; @@ -91,25 +71,6 @@ pub struct OverlayedValue { pub extrinsics: Option>, } -/// History of value that are related to a state history (eg field `history` of -/// an `OverlayedChangeSet`). -/// -/// Values are always paired with a state history index. -#[derive(Debug, Clone)] -#[cfg_attr(test, derive(PartialEq))] -pub(crate) struct History(SmallVec<[(V, usize); ALLOCATED_HISTORY]>); - -impl Default for History { - fn default() -> Self { - History(SmallVec::new()) - } -} - -/// Size of preallocated history per element. -/// Currently at two for committed and prospective only. -/// It means that using transaction in a module got a direct allocation cost. -const ALLOCATED_HISTORY: usize = 2; - /// Overlayed change set, keep history of values. /// /// This does not work by stacking hashmap for transaction, @@ -118,27 +79,17 @@ const ALLOCATED_HISTORY: usize = 2; /// When dropping or committing a layer of transaction, /// history for each values is kept until /// next mutable access to the value. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { /// Indexed state history. - pub(crate) history: Vec, + pub(crate) history: States, /// Top level storage changes. pub(crate) top: HashMap, History>, /// Child storage changes. pub(crate) children: HashMap, (HashMap, History>)>, } -impl Default for OverlayedChangeSet { - fn default() -> Self { - OverlayedChangeSet { - history: vec![TransactionState::Pending], - top: Default::default(), - children: Default::default(), - } - } -} - #[cfg(test)] impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { @@ -153,326 +104,60 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { } } -impl std::ops::Index for History { - type Output = (V, usize); - fn index(&self, index: usize) -> &Self::Output { - self.0.index(index) - } -} - -impl std::ops::IndexMut for History { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - self.0.index_mut(index) - } -} - -impl History { - - #[cfg(test)] - /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator) -> Self { - let mut history = History::default(); - for v in input { - history.push(v.0, v.1); - } - history - } - - fn len(&self) -> usize { - self.0.len() - } - - fn truncate(&mut self, index: usize) { - self.0.truncate(index) - } - - fn pop(&mut self) -> Option<(V, usize)> { - self.0.pop() - } - - // should only be call after a get_mut check - fn push(&mut self, val: V, tx_index: usize) { - self.0.push((val, tx_index)) - } - - /// Access to latest pending value (non dropped state in history). - /// When possible please prefer `get_mut` as it can free - /// some memory. - fn get(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Committed => - return Some(&self[index].0), - } - } - None - } - - #[cfg(test)] - fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => - return Some(&self[index].0), - TransactionState::Committed - | TransactionState::Dropped => (), - } - } - None - } - - #[cfg(test)] - fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => - return Some(&self[index].0), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), - } - } - None - } - - fn into_committed(mut self, history: &[TransactionState]) -> Option { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => { - self.truncate(index + 1); - return self.pop().map(|v| v.0); - }, - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), - } - } - None - } - - /// Access to latest pending value (non dropped state in history). - /// This method uses `get_mut` and do remove pending - /// This method remove latest dropped value up to the latest valid value. - fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { - let mut index = self.len(); - if index == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => { - // here we could gc all preceding values but that is additional cost - // and get_mut should stop at pending following committed. - let tx = self[index].1; - return Some((&mut self[index].0, tx)) - }, - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => { - let tx = self[index].1; - return Some((&mut self[index].0, tx)) - }, - TransactionState::Dropped => { let _ = self.pop(); }, - } - } - None - } - - /// Set a value, it uses a state history as parameter. - /// This method uses `get_mut` and do remove pending - /// dropped value. - fn set(&mut self, history: &[TransactionState], val: V) { - if let Some((v, index)) = self.get_mut(history) { - if index == history.len() - 1 { - *v = val; - return; - } - } - self.push(val, history.len() - 1); - } - - /// Garbage collect a history, act as a `get_mut` with additional cost. - /// If `eager` is true, all dropped value are removed even if it means shifting - /// array byte. Otherwhise we mainly ensure collection up to last Commit state - /// (truncate left size). - fn gc( - &mut self, - history: &[TransactionState], - tx_index: Option<&[usize]>, - ) -> Option<(&mut V, usize)> { - if let Some(tx_index) = tx_index { - let mut tx_index = &tx_index[..]; - let mut index = self.len(); - if index == 0 { - return None; - } - let mut bellow_value = usize::max_value(); - let mut result: Option = None; - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => { - for _ in 0..index { - let _ = self.0.remove(0); - } - result = Some(result.map(|i| i - index).unwrap_or(0)); - index = 0; - }, - TransactionState::Pending - | TransactionState::Prospective => { - if history_index >= bellow_value { - let _ = self.0.remove(index); - result.as_mut().map(|i| *i = *i - 1); - } else { - if result.is_none() { - result = Some(index); - } - while bellow_value > history_index { - // bellow_value = pop - let split = tx_index.split_last() - .map(|(v, sl)| (*v, sl)) - .unwrap_or((0, &[])); - bellow_value = split.0; - tx_index = split.1; - } - } - }, - TransactionState::TxPending => { - if history_index >= bellow_value { - let _ = self.0.remove(index); - result.as_mut().map(|i| *i = *i - 1); - } else { - if result.is_none() { - result = Some(index); - } - } - bellow_value = usize::max_value(); - }, - TransactionState::Dropped => { - let _ = self.0.remove(index); - }, - } - } - if let Some(index) = result { - let tx = self[index].1.clone(); - Some((&mut self[index].0, tx)) - } else { None } - } else { - return self.get_mut(history); - } +/// Variant of `set` value that update extrinsics. +/// It does remove latest dropped values. +fn set_with_extrinsic_overlayed_value( + h: &mut History, + history: &[TransactionState], + val: Option>, + extrinsic_index: Option, +) { + if let Some(extrinsic) = extrinsic_index { + set_with_extrinsic_inner_overlayed_value(h, history, val, extrinsic) + } else { + h.set(history, OverlayedValue { + value: val, + extrinsics: None, + }) } } -impl History { - - /// Variant of `set` value that update extrinsics. - /// It does remove latest dropped values. - fn set_with_extrinsic( - &mut self, history: &[TransactionState], - val: Option>, - extrinsic_index: Option, - ) { - if let Some(extrinsic) = extrinsic_index { - self.set_with_extrinsic_inner(history, val, extrinsic) - } else { - self.set(history, OverlayedValue { - value: val, - extrinsics: None, - }) - } - } +fn set_with_extrinsic_inner_overlayed_value( + h: &mut History, + history: &[TransactionState], + val: Option>, + extrinsic_index: u32, +) { + let state = history.len() - 1; + if let Some((mut current, current_index)) = h.get_mut(history) { + + if current_index == state { + current.value = val; + current.extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); - fn set_with_extrinsic_inner( - &mut self, history: &[TransactionState], - val: Option>, - extrinsic_index: u32, - ) { - let state = history.len() - 1; - if let Some((mut current, current_index)) = self.get_mut(history) { - - if current_index == state { - current.value = val; - current.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic_index); - - } else { - let mut extrinsics = current.extrinsics.clone(); - extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic_index); - self.push(OverlayedValue { - value: val, - extrinsics, - }, state); - } } else { - let mut extrinsics: Option> = None; + let mut extrinsics = current.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - - self.push(OverlayedValue { - value: val, - extrinsics, + h.force_push(OverlayedValue { + value: val, + extrinsics, }, state); - } - } + } else { + let mut extrinsics: Option> = None; + extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); + + h.force_push(OverlayedValue { + value: val, + extrinsics, + }, state); + } } impl OverlayedChangeSet { @@ -480,8 +165,8 @@ impl OverlayedChangeSet { fn gc(&mut self, eager: bool) { let eager = if eager { let mut tx_ixs = Vec::new(); - for (i, h) in self.history.iter().enumerate() { - if &TransactionState::TxPending == h { + for (i, h) in self.history.iter() { + if TransactionState::TxPending == h { tx_ixs.push(i); } } @@ -489,7 +174,7 @@ impl OverlayedChangeSet { } else { None }; - let history = self.history.as_slice(); + let history = self.history.as_ref(); // retain does change values self.top.retain(|_, h| h.gc(history, eager.as_ref().map(|t| t.as_slice())).is_some()); self.children.retain(|_, m| { @@ -506,86 +191,35 @@ impl OverlayedChangeSet { /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { - let mut i = self.history.len(); - while i > 0 { - i -= 1; - match self.history[i] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => self.history[i] = TransactionState::Dropped, - TransactionState::Committed => break, - } - } - self.history.push(TransactionState::Pending); + self.history.discard_prospective(); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - debug_assert!(self.history.len() > 0); - let mut i = self.history.len(); - while i > 0 { - i -= 1; - match self.history[i] { - TransactionState::Dropped => (), - TransactionState::Prospective - | TransactionState::TxPending - | TransactionState::Pending => self.history[i] = TransactionState::Committed, - | TransactionState::Committed => break, - } - } - self.history.push(TransactionState::Pending); + self.history.commit_prospective(); } /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.history.push(TransactionState::TxPending); + self.history.start_transaction(); } /// Discard a transactional layer. /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { - let mut i = self.history.len(); - while i > 0 { - i -= 1; - match self.history[i] { - TransactionState::Dropped => (), - TransactionState::Prospective - | TransactionState::Pending => self.history[i] = TransactionState::Dropped, - TransactionState::TxPending => { - self.history[i] = TransactionState::Dropped; - break; - }, - TransactionState::Committed => break, - } - } - self.history.push(TransactionState::Pending); + self.history.discard_transaction(); } /// Commit a transactional layer. pub fn commit_transaction(&mut self) { - let mut i = self.history.len(); - while i > 0 { - i -= 1; - match self.history[i] { - TransactionState::Prospective - | TransactionState::Dropped => (), - TransactionState::Pending => self.history[i] = TransactionState::Prospective, - TransactionState::TxPending => { - self.history[i] = TransactionState::Prospective; - break; - }, - TransactionState::Committed => break, - } - } - self.history.push(TransactionState::Pending); + self.history.commit_transaction(); } /// Iterator over current state of the overlay. pub fn top_iter_overlay(&self) -> impl Iterator { self.top.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()).map(|v| (k.as_slice(), v))) + v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) } /// Iterator over current state of the overlay. @@ -602,7 +236,7 @@ impl OverlayedChangeSet { .into_iter() .flat_map(move |child| child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()).map(|v| (k.as_slice(), v))) + v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) ) } @@ -618,7 +252,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()).map(|v| (k.as_slice(), v))) + v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) )) } @@ -629,7 +263,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()) + v.get(self.history.as_ref()) .map(|v| (k.as_slice(), v.value.as_ref().map(|v| v.as_slice())))) )) } @@ -641,7 +275,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.to_vec(), child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_slice()) + v.get(self.history.as_ref()) .map(|v| (k.to_vec(), v.value.as_ref().map(|v| v.to_vec())))) )) } @@ -653,7 +287,7 @@ impl OverlayedChangeSet { pub(crate) fn top_prospective(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_prospective(self.history.as_slice()) { + if let Some(v) = v.get_prospective(self.history.as_ref()) { result.insert(k.clone(), v.clone()); } } @@ -666,7 +300,7 @@ impl OverlayedChangeSet { pub(crate) fn top_committed(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_committed(self.history.as_slice()) { + if let Some(v) = v.get_committed(self.history.as_ref()) { result.insert(k.clone(), v.clone()); } } @@ -709,7 +343,7 @@ impl OverlayedChanges { /// value has been set. pub fn storage(&self, key: &[u8]) -> Option> { if let Some(overlay_val) = self.changes.top.get(key) { - if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { + if let Some(o_val) = overlay_val.get(self.changes.history.as_ref()) { return Some(o_val.value.as_ref().map(|v| v.as_slice())) } } @@ -722,7 +356,7 @@ impl OverlayedChanges { pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.changes.children.get(storage_key) { if let Some(overlay_val) = map.get(key) { - if let Some(o_val) = overlay_val.get(self.changes.history.as_slice()) { + if let Some(o_val) = overlay_val.get(self.changes.history.as_ref()) { return Some(o_val.value.as_ref().map(|v| v.as_slice())) } } @@ -744,7 +378,7 @@ impl OverlayedChanges { self.add_cost_op(&val); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - entry.set_with_extrinsic(self.changes.history.as_slice(), val, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), val, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -755,7 +389,7 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); - entry.set_with_extrinsic(self.changes.history.as_slice(), val, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), val, extrinsic_index); } /// Clear child storage of given storage key. @@ -766,11 +400,11 @@ impl OverlayedChanges { /// [`discard_prospective`]: #method.discard_prospective pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) { let extrinsic_index = self.extrinsic_index(); - let history = self.changes.history.as_slice(); + let history = self.changes.history.as_ref(); let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); self.operation_from_last_gc += map_entry.len(); - map_entry.values_mut().for_each(|e| e.set_with_extrinsic(history, None, extrinsic_index)); + map_entry.values_mut().for_each(|e| set_with_extrinsic_overlayed_value(e, history, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -786,7 +420,7 @@ impl OverlayedChanges { for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { nb_remove += 1; - entry.set_with_extrinsic(self.changes.history.as_slice(), None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), None, extrinsic_index); } } @@ -800,7 +434,7 @@ impl OverlayedChanges { for (key, entry) in child_change.iter_mut() { if key.starts_with(prefix) { nb_remove += 1; - entry.set_with_extrinsic(self.changes.history.as_slice(), None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), None, extrinsic_index); } } @@ -864,11 +498,11 @@ impl OverlayedChanges { let history2 = self.changes.history; ( top.into_iter() - .filter_map(move |(k, v)| v.into_committed(history.as_slice()).map(|v| (k, v.value))), + .filter_map(move |(k, v)| v.into_committed(history.as_ref()).map(|v| (k, v.value))), children.into_iter().map(move |(sk, v)| { let history2 = history2.clone(); (sk, v.into_iter() - .filter_map(move |(k, v)| v.into_committed(history2.as_slice()).map(|v| (k, v.value)))) + .filter_map(move |(k, v)| v.into_committed(history2.as_ref()).map(|v| (k, v.value)))) }) ) } @@ -923,12 +557,13 @@ impl OverlayedChanges { self.changes.top_iter() } + #[cfg(any(test, feature = "test"))] /// Count (slow) the number of key value, history included. /// Only for debugging or testing usage. pub fn top_count_keyvalue_pair(&self) -> usize { let mut result = 0; for (_, v) in self.changes.top.iter() { - result += v.0.len() + result += v.internal_item_counts() } result } diff --git a/core/test-runtime/Cargo.toml b/core/test-runtime/Cargo.toml index 56c9915192e0f..a9828f272c1f5 100644 --- a/core/test-runtime/Cargo.toml +++ b/core/test-runtime/Cargo.toml @@ -31,6 +31,7 @@ cfg-if = "0.1.6" srml-babe = { path = "../../srml/babe", default-features = false } srml-timestamp = { path = "../../srml/timestamp", default-features = false } srml-system = { path = "../../srml/system", default-features = false } +history-driven-data = { path = "../utils/history-driven-data", default-features = false } [dev-dependencies] substrate-executor = { path = "../executor" } @@ -71,4 +72,5 @@ std = [ "srml-system/std", "app-crypto/std", "session/std", + "history-driven-data/std", ] diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 175dd6f98a5a9..afb49c33eed31 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -257,6 +257,8 @@ cfg_if! { fn fail_on_wasm() -> u64; /// Trie no_std testing. fn use_trie() -> u64; + /// History data no_std testing. + fn use_history_data() -> u64; /// Transactional tests. fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; @@ -301,6 +303,8 @@ cfg_if! { fn fail_on_wasm() -> u64; /// trie no_std testing fn use_trie() -> u64; + /// History data no_std testing. + fn use_history_data() -> u64; /// Transactional tests. fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; @@ -441,6 +445,32 @@ fn code_using_trie() -> u64 { } else { 103 } } +fn history_driven_data() -> u64 { + + let mut states = history_driven_data::linear::States::default(); + let mut value = history_driven_data::linear::History::default(); + if value.get(states.as_ref()) != None { + return 101; + } + + value.set(states.as_ref(), 42u64); + states.start_transaction(); + if value.get(states.as_ref()) != Some(&42) { + return 102; + } + value.set(states.as_ref(), 43u64); + if value.get(states.as_ref()) != Some(&43) { + return 103; + } + states.discard_transaction(); + if value.get(states.as_ref()) != Some(&42) { + return 104; + } + + return 0; +} + + impl_opaque_keys! { pub struct SessionKeys { #[id(key_types::ED25519)] @@ -552,6 +582,10 @@ cfg_if! { code_using_trie() } + fn use_history_data() -> u64 { + history_driven_data() + } + fn use_transactions() -> u64 { system::test_transactions() } @@ -744,6 +778,10 @@ cfg_if! { code_using_trie() } + fn use_history_data() -> u64 { + history_driven_data() + } + fn use_transactions() -> u64 { system::test_transactions() } diff --git a/core/utils/history-driven-data/Cargo.toml b/core/utils/history-driven-data/Cargo.toml new file mode 100644 index 0000000000000..d6acc5a3496ff --- /dev/null +++ b/core/utils/history-driven-data/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "history-driven-data" +version = "2.0.0" +authors = ["Parity Technologies "] +description = "Data stored with per item history" +edition = "2018" + +[dependencies] +smallvec = { version = "0.6", optional = true } +rstd = { package = "sr-std", path = "../../sr-std", default-features = false } + +[dev-dependencies] + +[features] +default = ["std"] +std = [ + "smallvec", + "rstd/std", +] +test = [] diff --git a/core/utils/history-driven-data/README.md b/core/utils/history-driven-data/README.md new file mode 100644 index 0000000000000..b15df6dc654fe --- /dev/null +++ b/core/utils/history-driven-data/README.md @@ -0,0 +1,20 @@ +## History driven data + +Crate with methods to manage data that stores its own history. + +This covers: +- linear history driven data, eg. transactional layers for overlay. +- long term storage with multiple branch, eg. offchain storage. + +General design is container where query and update requires global +history context. + +History is serialize as a per item basis. + + +This crates should be `no_std` compatible, feature `std` should be use +for std usage. + +For more information see + +License: GPL-3.0 diff --git a/core/utils/history-driven-data/src/lib.rs b/core/utils/history-driven-data/src/lib.rs new file mode 100644 index 0000000000000..54ae5923028eb --- /dev/null +++ b/core/utils/history-driven-data/src/lib.rs @@ -0,0 +1,68 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! History driven data storage. + +#![cfg_attr(not(feature = "std"), no_std)] + +pub mod linear; + +#[derive(Debug, Clone, PartialEq, Copy)] +/// State of a transaction, the states are stored +/// in a linear indexed history. +pub enum State { + /// Overlay is under change and can still be dropped. + Pending, + /// Overlay is under change and can still be dropped. + /// This also mark the start of a transaction. + TxPending, + /// Information is committed, but can still be dropped + /// using `discard_prospective` or `discard_transaction` + /// of a parent transaction. + Prospective, + /// Committed is information that cannot be dropped. + Committed, + /// Information pointing to this indexed historic state should + /// not be returned and can be removed. + Dropped, +} + +pub const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { + trigger_transaction_gc: 100_000, + trigger_commit_gc: 10_000, + add_content_size_unit: 64, +}; + +pub struct GCConfiguration { + /// Treshold in number of operation before running a garbage colletion. + /// + /// Should be same as `TRIGGER_COMMIT_GC` or higher + /// (we most likely do not want lower as transaction are + /// possibly more frequent than commit). + trigger_transaction_gc: usize, + + /// Treshold in number of operation before running a garbage colletion + /// on a commit operation. + /// + /// We may want a lower value than for a transaction, even + /// a 1 if we want to do it between every operation. + trigger_commit_gc: usize, + + /// Used to count big content as multiple operations. + /// This is a number of octet. + /// Set to 0 to ignore. + add_content_size_unit: usize, +} diff --git a/core/utils/history-driven-data/src/linear.rs b/core/utils/history-driven-data/src/linear.rs new file mode 100644 index 0000000000000..9c7043b01a9e8 --- /dev/null +++ b/core/utils/history-driven-data/src/linear.rs @@ -0,0 +1,436 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Transactional overlay implementation. +//! +//! This follows a linear succession of states. +//! This contains multiple unbounded transaction layer +//! and an additional top level 'prospective' layer. +//! It only allows linear history (no branch so +//! inner storage is only an array of element). + +use crate::State as TransactionState; +use rstd::vec::Vec; +use rstd::vec; + +#[cfg(not(feature = "std"))] +type InnerVec = Vec<(V, usize)>; + +#[cfg(feature = "std")] +type InnerVec = smallvec::SmallVec<[(V, usize); ALLOCATED_HISTORY]>; + +/// Size of preallocated history per element. +/// Currently at two for committed and prospective only. +/// It means that using transaction in a module got a direct allocation cost. +const ALLOCATED_HISTORY: usize = 2; + +/// History of value that are related to a state history (eg field `history` of +/// an `OverlayedChangeSet`). +/// +/// Values are always paired with a state history index. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct History(InnerVec); + +impl Default for History { + fn default() -> Self { + History(Default::default()) + } +} + +impl rstd::ops::Index for History { + type Output = (V, usize); + fn index(&self, index: usize) -> &Self::Output { + self.0.index(index) + } +} + +impl rstd::ops::IndexMut for History { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + self.0.index_mut(index) + } +} + +impl History { + + #[cfg(any(test, feature = "test"))] + /// Create an history from an existing history. + pub fn from_iter(input: impl IntoIterator) -> Self { + let mut history = History::default(); + for v in input { + history.push(v.0, v.1); + } + history + } + + #[cfg(any(test, feature = "test"))] + /// Debugging function for test and fuzzing. + pub fn internal_item_counts(&self) -> usize { + self.0.len() + } + + fn len(&self) -> usize { + self.0.len() + } + + fn truncate(&mut self, index: usize) { + self.0.truncate(index) + } + + fn pop(&mut self) -> Option<(V, usize)> { + self.0.pop() + } + + /// Append without checking if a value already exist. + /// If a value already exists, the history will be broken. + /// This method shall only be call after a `get_mut` where + /// the returned index indicate that a `set` will result + /// in appending a value. + pub fn force_push(&mut self, val: V, tx_index: usize) { + self.0.push((val, tx_index)) + } + + /// Access to latest pending value (non dropped state in history). + /// When possible please prefer `get_mut` as it can free + /// some memory. + pub fn get(&self, history: &[TransactionState]) -> Option<&V> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Committed => + return Some(&self[index].0), + } + } + None + } + + #[cfg(any(test, feature = "test"))] + pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => + return Some(&self[index].0), + TransactionState::Committed + | TransactionState::Dropped => (), + } + } + None + } + + #[cfg(any(test, feature = "test"))] + pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Committed => + return Some(&self[index].0), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + pub fn into_committed(mut self, history: &[TransactionState]) -> Option { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Committed => { + self.truncate(index + 1); + return self.pop().map(|v| v.0); + }, + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + /// Access to latest pending value (non dropped state in history). + /// This method uses `get_mut` and do remove pending + /// This method remove latest dropped value up to the latest valid value. + pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { + + let mut index = self.len(); + if index == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Committed => { + // here we could gc all preceding values but that is additional cost + // and get_mut should stop at pending following committed. + let tx = self[index].1; + return Some((&mut self[index].0, tx)) + }, + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => { + let tx = self[index].1; + return Some((&mut self[index].0, tx)) + }, + TransactionState::Dropped => { let _ = self.pop(); }, + } + } + None + } + + /// Set a value, it uses a state history as parameter. + /// This method uses `get_mut` and do remove pending + /// dropped value. + pub fn set(&mut self, history: &[TransactionState], val: V) { + if let Some((v, index)) = self.get_mut(history) { + if index == history.len() - 1 { + *v = val; + return; + } + } + self.force_push(val, history.len() - 1); + } + + /// Garbage collect a history, act as a `get_mut` with additional cost. + /// If `eager` is true, all dropped value are removed even if it means shifting + /// array byte. Otherwhise we mainly ensure collection up to last Commit state + /// (truncate left size). + pub fn gc( + &mut self, + history: &[TransactionState], + tx_index: Option<&[usize]>, + ) -> Option<(&mut V, usize)> { + if let Some(tx_index) = tx_index { + let mut tx_index = &tx_index[..]; + let mut index = self.len(); + if index == 0 { + return None; + } + let mut bellow_value = usize::max_value(); + let mut result: Option = None; + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self[index].1; + match history[history_index] { + TransactionState::Committed => { + for _ in 0..index { + let _ = self.0.remove(0); + } + result = Some(result.map(|i| i - index).unwrap_or(0)); + index = 0; + }, + TransactionState::Pending + | TransactionState::Prospective => { + if history_index >= bellow_value { + let _ = self.0.remove(index); + result.as_mut().map(|i| *i = *i - 1); + } else { + if result.is_none() { + result = Some(index); + } + while bellow_value > history_index { + // bellow_value = pop + let split = tx_index.split_last() + .map(|(v, sl)| (*v, sl)) + .unwrap_or((0, &[])); + bellow_value = split.0; + tx_index = split.1; + } + } + }, + TransactionState::TxPending => { + if history_index >= bellow_value { + let _ = self.0.remove(index); + result.as_mut().map(|i| *i = *i - 1); + } else { + if result.is_none() { + result = Some(index); + } + } + bellow_value = usize::max_value(); + }, + TransactionState::Dropped => { + let _ = self.0.remove(index); + }, + } + } + if let Some(index) = result { + let tx = self[index].1.clone(); + Some((&mut self[index].0, tx)) + } else { None } + + } else { + return self.get_mut(history); + } + } +} + + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct States(Vec); + +impl Default for States { + fn default() -> Self { + States(vec![TransactionState::Pending]) + } +} + +impl States { + pub fn as_ref(&self) -> &[TransactionState] { + self.0.as_ref() + } + + pub fn iter<'a>(&'a self) -> impl Iterator + 'a { + self.0.iter().map(Clone::clone).enumerate() + } +} + +impl States { + + /// Build any state for testing only. + #[cfg(any(test, feature = "test"))] + pub fn test_vector(test_state: Vec) -> Self { + States(test_state) + } + + /// Discard prospective changes to state. + pub fn discard_prospective(&mut self) { + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => self.0[i] = TransactionState::Dropped, + TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + + /// Commit prospective changes to state. + pub fn commit_prospective(&mut self) { + debug_assert!(self.0.len() > 0); + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Dropped => (), + TransactionState::Prospective + | TransactionState::TxPending + | TransactionState::Pending => self.0[i] = TransactionState::Committed, + | TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.0.push(TransactionState::TxPending); + } + + /// Discard a transactional layer. + /// A transaction is always running (history always end with pending). + pub fn discard_transaction(&mut self) { + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Dropped => (), + TransactionState::Prospective + | TransactionState::Pending => self.0[i] = TransactionState::Dropped, + TransactionState::TxPending => { + self.0[i] = TransactionState::Dropped; + break; + }, + TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Prospective + | TransactionState::Dropped => (), + TransactionState::Pending => self.0[i] = TransactionState::Prospective, + TransactionState::TxPending => { + self.0[i] = TransactionState::Prospective; + break; + }, + TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + +} From 75fd7983eb82617bcfc9ec84304cf51d840acfb2 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 26 Aug 2019 16:18:22 +0200 Subject: [PATCH 028/129] Make history buffer parametric (to allow encoding efficient implementation). --- core/state-machine/src/ext.rs | 1 - core/state-machine/src/overlayed_changes.rs | 6 +- core/test-runtime/src/lib.rs | 4 +- core/utils/history-driven-data/src/linear.rs | 62 +++++++++++++++++--- 4 files changed, 60 insertions(+), 13 deletions(-) diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 8b7317b3d8f43..fc2160db54fb1 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -377,7 +377,6 @@ mod tests { InMemoryStorage as InMemoryChangesTrieStorage}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; use history_driven_data::linear::History; - use history_driven_data::State as TransactionState; use super::*; type TestBackend = InMemory; diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 5273e742e8784..6eacb38d63bf3 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -21,9 +21,11 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use history_driven_data::linear::{States, History}; +use history_driven_data::linear::{States, History as HistoryBasis, MemoryOnly}; use history_driven_data::State as TransactionState; +type History = HistoryBasis, V>; + /// Treshold in number of operation before running a garbage colletion. /// /// Should be same as `TRIGGER_COMMIT_GC` or higher @@ -97,7 +99,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, v)| (k, { let mut history = History::default(); - history.push(v, 0); + history.force_push(v, 0); history })).collect(); result diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index afb49c33eed31..ac525b1f73d1f 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -448,7 +448,9 @@ fn code_using_trie() -> u64 { fn history_driven_data() -> u64 { let mut states = history_driven_data::linear::States::default(); - let mut value = history_driven_data::linear::History::default(); + let mut value = history_driven_data::linear::History::< + history_driven_data::linear::MemoryOnly<_>, + _>::default(); if value.get(states.as_ref()) != None { return 101; } diff --git a/core/utils/history-driven-data/src/linear.rs b/core/utils/history-driven-data/src/linear.rs index 9c7043b01a9e8..0896b4a822458 100644 --- a/core/utils/history-driven-data/src/linear.rs +++ b/core/utils/history-driven-data/src/linear.rs @@ -25,12 +25,19 @@ use crate::State as TransactionState; use rstd::vec::Vec; use rstd::vec; +use rstd::marker::PhantomData; +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. #[cfg(not(feature = "std"))] -type InnerVec = Vec<(V, usize)>; +pub type MemoryOnly = Vec<(V, usize)>; +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. #[cfg(feature = "std")] -type InnerVec = smallvec::SmallVec<[(V, usize); ALLOCATED_HISTORY]>; +pub type MemoryOnly = smallvec::SmallVec<[(V, usize); ALLOCATED_HISTORY]>; /// Size of preallocated history per element. /// Currently at two for committed and prospective only. @@ -43,35 +50,72 @@ const ALLOCATED_HISTORY: usize = 2; /// Values are always paired with a state history index. #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] -pub struct History(InnerVec); +pub struct History(B, PhantomData); + +trait HistoryBuffer: Default + + rstd::ops::Index + + rstd::ops::IndexMut +{ + + fn len(&self) -> usize; + fn truncate(&mut self, _index: usize); + fn pop(&mut self) -> Option<(V, usize)>; + fn push(&mut self, _val: (V, usize)); + fn remove(&mut self, _index: usize) -> (V, usize); +} + +impl HistoryBuffer for MemoryOnly { + fn len(&self) -> usize { + self.len() + } + + fn truncate(&mut self, index: usize) { + self.truncate(index) + } + + fn pop(&mut self) -> Option<(V, usize)> { + self.pop() + } + + fn push(&mut self, val: (V, usize)) { + self.push(val) + } + + fn remove(&mut self, index: usize) -> (V, usize) { + self.remove(index) + } + + + +} -impl Default for History { +impl, V> Default for History { fn default() -> Self { - History(Default::default()) + History(Default::default(), PhantomData) } } -impl rstd::ops::Index for History { +impl, V> rstd::ops::Index for History { type Output = (V, usize); fn index(&self, index: usize) -> &Self::Output { self.0.index(index) } } -impl rstd::ops::IndexMut for History { +impl, V> rstd::ops::IndexMut for History { fn index_mut(&mut self, index: usize) -> &mut Self::Output { self.0.index_mut(index) } } -impl History { +impl, V> History { #[cfg(any(test, feature = "test"))] /// Create an history from an existing history. pub fn from_iter(input: impl IntoIterator) -> Self { let mut history = History::default(); for v in input { - history.push(v.0, v.1); + history.force_push(v.0, v.1); } history } From 1fb154e114912de48bf9d90f6de521c5e302b6f6 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Tue, 27 Aug 2019 19:53:15 +0200 Subject: [PATCH 029/129] Removed history trait, to go to the more pragmatic macro approach (after next testing step). --- Cargo.lock | 6 +- Cargo.toml | 2 +- core/sr-std/without_std.rs | 2 +- core/state-machine/Cargo.toml | 8 +- core/state-machine/src/changes_trie/build.rs | 4 +- core/state-machine/src/ext.rs | 2 +- core/state-machine/src/overlayed_changes.rs | 47 +- core/test-runtime/Cargo.toml | 4 +- core/test-runtime/src/lib.rs | 12 +- .../Cargo.toml | 8 +- .../README.md | 8 +- .../src/lib.rs | 40 +- core/utils/historied-data/src/linear.rs | 838 ++++++++++++++++++ core/utils/history-driven-data/src/linear.rs | 480 ---------- 14 files changed, 902 insertions(+), 559 deletions(-) rename core/utils/{history-driven-data => historied-data}/Cargo.toml (81%) rename core/utils/{history-driven-data => historied-data}/README.md (66%) rename core/utils/{history-driven-data => historied-data}/src/lib.rs (68%) create mode 100644 core/utils/historied-data/src/linear.rs delete mode 100644 core/utils/history-driven-data/src/linear.rs diff --git a/Cargo.lock b/Cargo.lock index f9c3896063e32..2bb225393df88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1213,7 +1213,7 @@ dependencies = [ ] [[package]] -name = "history-driven-data" +name = "historied-data" version = "2.0.0" dependencies = [ "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5130,7 +5130,7 @@ dependencies = [ "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "history-driven-data 2.0.0", + "historied-data 2.0.0", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5186,7 +5186,7 @@ name = "substrate-test-runtime" version = "2.0.0" dependencies = [ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "history-driven-data 2.0.0", + "historied-data 2.0.0", "log 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 91e26823fecaf..e5a5ca5351863 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ members = [ "core/transaction-pool/graph", "core/trie", "core/utils/fork-tree", - "core/utils/history-driven-data", + "core/utils/historied-data", "core/utils/wasm-builder", "core/utils/wasm-builder-runner", "srml/support", diff --git a/core/sr-std/without_std.rs b/core/sr-std/without_std.rs index 327e271049d58..75483e4199d83 100755 --- a/core/sr-std/without_std.rs +++ b/core/sr-std/without_std.rs @@ -49,7 +49,7 @@ mod __impl { pub use alloc::boxed; pub use alloc::rc; pub use alloc::vec; -pub use core::borrow; +pub use alloc::borrow; pub use core::cell; pub use core::clone; pub use core::cmp; diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index cfeb687f6beb8..da0c8dc8c7a23 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -16,13 +16,13 @@ primitives = { package = "substrate-primitives", path = "../primitives" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } codec = { package = "parity-scale-codec", version = "1.0.0" } num-traits = "0.2" -history-driven-data = { path = "../utils/history-driven-data", features = ["std"] } +historied-data = { path = "../utils/historied-data", features = ["std"] } [dev-dependencies] hex-literal = "0.2.0" criterion = "0.2" rand = "0.6" -history-driven-data = { path = "../utils/history-driven-data", features = ["std", "test"] } +historied-data = { path = "../utils/historied-data", features = ["std", "test"] } [[bench]] name = "bench" @@ -30,4 +30,6 @@ harness = false [features] default = [] -test = [] +test = [ + "historied-data/test", +] diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 85264e38c7ed1..ea6659c2735e6 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -178,8 +178,8 @@ mod test { use crate::backend::InMemory; use crate::changes_trie::storage::InMemoryStorage; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use history_driven_data::linear::{History, States}; - use history_driven_data::State as TransactionState; + use historied_data::linear::{History, States}; + use historied_data::State as TransactionState; use super::*; fn prepare_for_build() -> (InMemory, InMemoryStorage, OverlayedChanges) { diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index fc2160db54fb1..eb8914f6d35b1 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -376,7 +376,7 @@ mod tests { use crate::changes_trie::{Configuration as ChangesTrieConfiguration, InMemoryStorage as InMemoryChangesTrieStorage}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use history_driven_data::linear::History; + use historied_data::linear::History; use super::*; type TestBackend = InMemory; diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 6eacb38d63bf3..0c5dc80fa4a66 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -21,29 +21,9 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use history_driven_data::linear::{States, History as HistoryBasis, MemoryOnly}; -use history_driven_data::State as TransactionState; - -type History = HistoryBasis, V>; - -/// Treshold in number of operation before running a garbage colletion. -/// -/// Should be same as `TRIGGER_COMMIT_GC` or higher -/// (we most likely do not want lower as transaction are -/// possibly more frequent than commit). -const TRIGGER_TRANSACTION_GC: usize = 100_000; - -/// Treshold in number of operation before running a garbage colletion -/// on a commit operation. -/// -/// We may want a lower value than for a transaction, even -/// a 1 if we want to do it between every operation. -const TRIGGER_COMMIT_GC: usize = 10_000; - -/// Used to count big content as multiple operations. -/// This is a number of octet. -/// Set to 0 to ignore. -const ADD_CONTENT_SIZE_UNIT: usize = 64; +use historied_data::linear::{States, History}; +use historied_data::State as TransactionState; +use historied_data::DEFAULT_GC_CONF; /// The overlayed changes to state to be queried on top of the backend. /// @@ -366,18 +346,11 @@ impl OverlayedChanges { None } - fn add_cost_op(&mut self, val: &Option>) { - let content_cost = if ADD_CONTENT_SIZE_UNIT > 0 { - val.as_ref().map(|s| s.len() / ADD_CONTENT_SIZE_UNIT).unwrap_or(0) - } else { 0 }; - self.operation_from_last_gc += 1 + content_cost; - } - /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. pub fn set_storage(&mut self, key: Vec, val: Option>) { - self.add_cost_op(&val); + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&val); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), val, extrinsic_index); @@ -387,7 +360,7 @@ impl OverlayedChanges { /// /// `None` can be used to delete a value specified by the given key. pub(crate) fn set_child_storage(&mut self, storage_key: Vec, key: Vec, val: Option>) { - self.add_cost_op(&val); + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&val); let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); @@ -447,7 +420,7 @@ impl OverlayedChanges { /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.changes.discard_prospective(); - if self.operation_from_last_gc > TRIGGER_COMMIT_GC { + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -456,7 +429,7 @@ impl OverlayedChanges { /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.changes.commit_prospective(); - if self.operation_from_last_gc > TRIGGER_COMMIT_GC { + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -465,7 +438,7 @@ impl OverlayedChanges { /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.changes.start_transaction(); - if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -475,7 +448,7 @@ impl OverlayedChanges { /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { self.changes.discard_transaction(); - if self.operation_from_last_gc > TRIGGER_TRANSACTION_GC { + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { self.operation_from_last_gc = 0; self.gc(true); } @@ -536,8 +509,6 @@ impl OverlayedChanges { result } - - /// Returns current extrinsic index to use in changes trie construction. /// None is returned if it is not set or changes trie config is not set. /// Persistent value (from the backend) can be ignored because runtime must diff --git a/core/test-runtime/Cargo.toml b/core/test-runtime/Cargo.toml index a9828f272c1f5..efbea80564d24 100644 --- a/core/test-runtime/Cargo.toml +++ b/core/test-runtime/Cargo.toml @@ -31,7 +31,7 @@ cfg-if = "0.1.6" srml-babe = { path = "../../srml/babe", default-features = false } srml-timestamp = { path = "../../srml/timestamp", default-features = false } srml-system = { path = "../../srml/system", default-features = false } -history-driven-data = { path = "../utils/history-driven-data", default-features = false } +historied-data = { path = "../utils/historied-data", default-features = false } [dev-dependencies] substrate-executor = { path = "../executor" } @@ -72,5 +72,5 @@ std = [ "srml-system/std", "app-crypto/std", "session/std", - "history-driven-data/std", + "historied-data/std", ] diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index ac525b1f73d1f..8916fc533f634 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -445,12 +445,10 @@ fn code_using_trie() -> u64 { } else { 103 } } -fn history_driven_data() -> u64 { +fn historied_data() -> u64 { - let mut states = history_driven_data::linear::States::default(); - let mut value = history_driven_data::linear::History::< - history_driven_data::linear::MemoryOnly<_>, - _>::default(); + let mut states = historied_data::linear::States::default(); + let mut value = historied_data::linear::History::default(); if value.get(states.as_ref()) != None { return 101; } @@ -585,7 +583,7 @@ cfg_if! { } fn use_history_data() -> u64 { - history_driven_data() + historied_data() } fn use_transactions() -> u64 { @@ -781,7 +779,7 @@ cfg_if! { } fn use_history_data() -> u64 { - history_driven_data() + historied_data() } fn use_transactions() -> u64 { diff --git a/core/utils/history-driven-data/Cargo.toml b/core/utils/historied-data/Cargo.toml similarity index 81% rename from core/utils/history-driven-data/Cargo.toml rename to core/utils/historied-data/Cargo.toml index d6acc5a3496ff..79ab10dd8658b 100644 --- a/core/utils/history-driven-data/Cargo.toml +++ b/core/utils/historied-data/Cargo.toml @@ -1,20 +1,20 @@ [package] -name = "history-driven-data" +name = "historied-data" version = "2.0.0" authors = ["Parity Technologies "] -description = "Data stored with per item history" +description = "Data associated with its history" edition = "2018" [dependencies] -smallvec = { version = "0.6", optional = true } rstd = { package = "sr-std", path = "../../sr-std", default-features = false } +smallvec = { version = "0.6", optional = true } [dev-dependencies] [features] default = ["std"] std = [ - "smallvec", "rstd/std", + "smallvec", ] test = [] diff --git a/core/utils/history-driven-data/README.md b/core/utils/historied-data/README.md similarity index 66% rename from core/utils/history-driven-data/README.md rename to core/utils/historied-data/README.md index b15df6dc654fe..8d469fbb00b56 100644 --- a/core/utils/history-driven-data/README.md +++ b/core/utils/historied-data/README.md @@ -1,4 +1,4 @@ -## History driven data +## Historied data Crate with methods to manage data that stores its own history. @@ -11,10 +11,8 @@ history context. History is serialize as a per item basis. +This crates is be `no_std` compatible, unless feature `std` is used. -This crates should be `no_std` compatible, feature `std` should be use -for std usage. - -For more information see +For more information see License: GPL-3.0 diff --git a/core/utils/history-driven-data/src/lib.rs b/core/utils/historied-data/src/lib.rs similarity index 68% rename from core/utils/history-driven-data/src/lib.rs rename to core/utils/historied-data/src/lib.rs index 54ae5923028eb..33a0ef7022f32 100644 --- a/core/utils/history-driven-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -15,27 +15,28 @@ // along with Substrate. If not, see . //! History driven data storage. +//! Useful to store information with history +//! on a per item basis. #![cfg_attr(not(feature = "std"), no_std)] pub mod linear; #[derive(Debug, Clone, PartialEq, Copy)] -/// State of a transaction, the states are stored -/// in a linear indexed history. +/// State of a data. pub enum State { - /// Overlay is under change and can still be dropped. + /// Data is under change and can still be dropped. Pending, - /// Overlay is under change and can still be dropped. + /// Data is under change and can still be dropped. /// This also mark the start of a transaction. TxPending, - /// Information is committed, but can still be dropped + /// Data is committed, but can still be dropped /// using `discard_prospective` or `discard_transaction` - /// of a parent transaction. + /// from a parent transaction state. Prospective, - /// Committed is information that cannot be dropped. + /// Committed is data that cannot be dropped. Committed, - /// Information pointing to this indexed historic state should + /// Data pointing to this indexed historic state should /// not be returned and can be removed. Dropped, } @@ -47,22 +48,37 @@ pub const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { }; pub struct GCConfiguration { - /// Treshold in number of operation before running a garbage colletion. + /// Treshold in number of operation before running a garbage collection. /// /// Should be same as `TRIGGER_COMMIT_GC` or higher /// (we most likely do not want lower as transaction are /// possibly more frequent than commit). - trigger_transaction_gc: usize, + pub trigger_transaction_gc: usize, /// Treshold in number of operation before running a garbage colletion /// on a commit operation. /// /// We may want a lower value than for a transaction, even /// a 1 if we want to do it between every operation. - trigger_commit_gc: usize, + pub trigger_commit_gc: usize, /// Used to count big content as multiple operations. /// This is a number of octet. /// Set to 0 to ignore. - add_content_size_unit: usize, + pub add_content_size_unit: usize, +} + +impl GCConfiguration { + /// Cost depending on value if any. + pub fn operation_cost(&self, val: &Option>) -> usize { + let additional_cost = if self.add_content_size_unit > 0 { + if let Some(s) = val.as_ref() { + s.len() / self.add_content_size_unit + } else { + 0 + } + } else { 0 }; + 1 + additional_cost + } + } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs new file mode 100644 index 0000000000000..a4af2574eef3c --- /dev/null +++ b/core/utils/historied-data/src/linear.rs @@ -0,0 +1,838 @@ +// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Transactional overlay implementation. +//! +//! This follows a linear succession of states. +//! This contains multiple unbounded transaction layer +//! and an additional top level 'prospective' layer. +//! It only allows linear history (no branch so +//! inner storage is only an array of element). + +use crate::State as TransactionState; +use rstd::vec::Vec; +use rstd::vec; +use rstd::borrow::Cow; + +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. +#[cfg(not(feature = "std"))] +type MemoryOnly = Vec<(V, usize)>; + +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. +#[cfg(feature = "std")] +type MemoryOnly = smallvec::SmallVec<[(V, usize); ALLOCATED_HISTORY]>; + +/// Size of preallocated history per element. +/// Currently at two for committed and prospective only. +/// It means that using transaction in a module got a direct allocation cost. +const ALLOCATED_HISTORY: usize = 2; + +/// History of value that are related to a state history (eg field `history` of +/// an `OverlayedChangeSet`). +/// +/// Values are always paired with a state history index. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct History(MemoryOnly); + + +impl History { + fn get_state(&self, index: usize) -> (&V, usize) { + (&self.0[index].0, self.0[index].1) + } +} + +impl Default for History { + fn default() -> Self { + History(Default::default()) + } +} + +impl History { + + /// Access to latest pending value (non dropped state in history). + /// When possible please prefer `get_mut` as it can free + /// some memory. + pub fn get(&self, history: &[TransactionState]) -> Option<&V> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let (v, history_index) = self.get_state(index); + match history[history_index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Committed => + return Some(v), + } + } + None + } + + #[cfg(any(test, feature = "test"))] + pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let (v, history_index) = self.get_state(index); + match history[history_index] { + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => + return Some(v), + TransactionState::Committed + | TransactionState::Dropped => (), + } + } + None + } + + #[cfg(any(test, feature = "test"))] + pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let (v, history_index) = self.get_state(index); + match history[history_index] { + TransactionState::Committed => + return Some(v), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + pub fn into_committed(mut self, history: &[TransactionState]) -> Option { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).1; + match history[history_index] { + TransactionState::Committed => { + self.truncate(index + 1); + return self.pop().map(|v| v.0); + }, + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + /// Access to latest pending value (non dropped state in history). + /// This method uses `get_mut` and do remove pending + /// This method remove latest dropped value up to the latest valid value. + pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { + + let mut index = self.len(); + if index == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).1; + match history[history_index] { + TransactionState::Committed => { + // here we could gc all preceding values but that is additional cost + // and get_mut should stop at pending following committed. + return Some((self.mut_ref(index), history_index)) + }, + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => { + return Some((self.mut_ref(index), history_index)) + }, + TransactionState::Dropped => { let _ = self.pop(); }, + } + } + None + } + + + /// Garbage collect a history, act as a `get_mut` with additional cost. + /// If `eager` is true, all dropped value are removed even if it means shifting + /// array byte. Otherwhise we mainly ensure collection up to last Commit state + /// (truncate left size). + pub fn gc( + &mut self, + history: &[TransactionState], + tx_index: Option<&[usize]>, + ) -> Option<(&mut V, usize)> { + if let Some(tx_index) = tx_index { + let mut tx_index = &tx_index[..]; + let mut index = self.len(); + if index == 0 { + return None; + } + let mut bellow_value = usize::max_value(); + let mut result: Option<(usize, usize)> = None; + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).1; + match history[history_index] { + TransactionState::Committed => { + for _ in 0..index { + self.remove(0); + } + result = Some(result.map(|(i, hi)| (i - index, hi)).unwrap_or((0, history_index))); + index = 0; + }, + TransactionState::Pending + | TransactionState::Prospective => { + if history_index >= bellow_value { + self.remove(index); + result.as_mut().map(|(i, _hi)| *i = *i - 1); + } else { + if result.is_none() { + result = Some((index, history_index)); + } + while bellow_value > history_index { + // bellow_value = pop + let split = tx_index.split_last() + .map(|(v, sl)| (*v, sl)) + .unwrap_or((0, &[])); + bellow_value = split.0; + tx_index = split.1; + } + } + }, + TransactionState::TxPending => { + if history_index >= bellow_value { + self.remove(index); + result.as_mut().map(|(i, _hi)| *i = *i - 1); + } else { + if result.is_none() { + result = Some((index, history_index)); + } + } + bellow_value = usize::max_value(); + }, + TransactionState::Dropped => { + self.remove(index); + }, + } + } + if let Some((index, history_index)) = result { + Some((self.mut_ref(index), history_index)) + } else { None } + + } else { + return self.get_mut(history); + } + } +} + + +impl History { + + #[cfg(any(test, feature = "test"))] + /// Create an history from an existing history. + pub fn from_iter(input: impl IntoIterator) -> Self { + let mut history = History::default(); + for v in input { + history.force_push(v.0, v.1); + } + history + } + + #[cfg(any(test, feature = "test"))] + /// Debugging function for test and fuzzing. + pub fn internal_item_counts(&self) -> usize { + self.0.len() + } + + fn len(&self) -> usize { + self.0.len() + } + + fn truncate(&mut self, index: usize) { + self.0.truncate(index) + } + + fn pop(&mut self) -> Option<(V, usize)> { + self.0.pop() + } + + fn remove(&mut self, index: usize) { + let _ = self.0.remove(index); + } + + /// Append without checking if a value already exist. + /// If a value already exists, the history will be broken. + /// This method shall only be call after a `get_mut` where + /// the returned index indicate that a `set` will result + /// in appending a value. + pub fn force_push(&mut self, val: V, tx_index: usize) { + self.0.push((val, tx_index)) + } + + /// Set a value, it uses a state history as parameter. + /// This method uses `get_mut` and do remove pending + /// dropped value. + pub fn set(&mut self, history: &[TransactionState], val: V) { + if let Some((v, index)) = self.get_mut(history) { + if index == history.len() - 1 { + *v = val; + return; + } + } + self.force_push(val, history.len() - 1); + } + + fn mut_ref(&mut self, index: usize) -> &mut V { + &mut self.0[index].0 + } + +} + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct States(Vec); + +impl Default for States { + fn default() -> Self { + States(vec![TransactionState::Pending]) + } +} + +impl States { + pub fn as_ref(&self) -> &[TransactionState] { + self.0.as_ref() + } + + pub fn iter<'a>(&'a self) -> impl Iterator + 'a { + self.0.iter().map(Clone::clone).enumerate() + } +} + +impl States { + + /// Build any state for testing only. + #[cfg(any(test, feature = "test"))] + pub fn test_vector(test_state: Vec) -> Self { + States(test_state) + } + + /// Discard prospective changes to state. + pub fn discard_prospective(&mut self) { + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => self.0[i] = TransactionState::Dropped, + TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + + /// Commit prospective changes to state. + pub fn commit_prospective(&mut self) { + debug_assert!(self.0.len() > 0); + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Dropped => (), + TransactionState::Prospective + | TransactionState::TxPending + | TransactionState::Pending => self.0[i] = TransactionState::Committed, + | TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.0.push(TransactionState::TxPending); + } + + /// Discard a transactional layer. + /// A transaction is always running (history always end with pending). + pub fn discard_transaction(&mut self) { + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Dropped => (), + TransactionState::Prospective + | TransactionState::Pending => self.0[i] = TransactionState::Dropped, + TransactionState::TxPending => { + self.0[i] = TransactionState::Dropped; + break; + }, + TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + let mut i = self.0.len(); + while i > 0 { + i -= 1; + match self.0[i] { + TransactionState::Prospective + | TransactionState::Dropped => (), + TransactionState::Pending => self.0[i] = TransactionState::Prospective, + TransactionState::TxPending => { + self.0[i] = TransactionState::Prospective; + break; + }, + TransactionState::Committed => break, + } + } + self.0.push(TransactionState::Pending); + } + +} + + +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +/// Arraylike buffer with in place byte data. +/// Can be written as is in underlying +/// storage. +/// Could be use for direct access memory to. +pub struct Serialized<'a>(Cow<'a, [u8]>); + +impl<'a> Serialized<'a> { + + fn len(&self) -> usize { + let len = self.0.len(); + self.read_le_usize(len - 4) + } + + fn truncate(&mut self, index: usize) { + let len = self.len(); + if index >= len { + return; + } + let start = self.index_start(); + let end_ix = self.index_element(index + 1); + let len_ix = len * 4; + self.slice_copy(start, end_ix, len_ix); + self.write_le_usize(end_ix + len_ix - 4, index); + self.0.to_mut().truncate(end_ix + len_ix); + } + + fn pop(&mut self) -> Option<(Vec, usize)> { + let len = self.len(); + if len == 0 { + return None; + } + let start_ix = self.index_element(len); + let end_ix = self.index_start(); + let state = self.read_le_usize(start_ix); + let value = self.0[start_ix + 4..end_ix].to_vec(); + self.write_le_usize(len - 1, self.0.len() - 8); + let ix_size = len * 4; + self.slice_copy(end_ix, start_ix, ix_size); + self.0.to_mut().truncate(start_ix + ix_size); + Some((value, state)) + } + + fn push(&mut self, val: (&[u8], usize)) { + let len = self.len(); + let start_ix = self.index_start(); + let end_ix = self.0.len(); + // A sized buffer and multiple index to avoid to big copy + // should be use here. + let mut new_ix = self.0[start_ix..end_ix].to_vec(); + // truncate here can be bad + self.0.to_mut().truncate(start_ix + 4); + self.write_le_usize(start_ix, val.1); + self.0.to_mut().extend_from_slice(val.0); + self.0.to_mut().append(&mut new_ix); + self.write_le_usize(self.0.len() - 4, start_ix); + self.append_le_usize(len + 1); + } + + fn remove(&mut self, index: usize) { + let len = self.len(); + // eager removal is costy, running some gc impl + // can be interesting (would be malleable serializing). + let elt_start = self.index_element(index); + let start_ix = self.index_start(); + let elt_end = if index == len { + start_ix + } else { + self.index_element(index) + }; + let delete_size = elt_end - elt_start; + for i in elt_start..elt_end { + let _ = self.0.to_mut().remove(i); + } + let start_ix = start_ix - delete_size; + for i in 0..len - index - 1 { + let old_value = self.read_le_usize(4 + start_ix + i * 4); + self.write_le_usize(start_ix + i * 4, old_value - delete_size); + } + let size_index = start_ix + (len - 1) * 4; + self.write_le_usize(size_index, len - 1); + self.0.to_mut().truncate(size_index + 4); + + } + + fn get_state(&self, index: usize) -> (&[u8], usize) { + let start_ix = self.index_element(index); + let len = self.len(); + let end_ix = if index == len - 1 { + self.index_start() + } else { + self.index_element(index + 1) + }; + let state = self.read_le_usize(start_ix); + (&self.0[start_ix..end_ix], state) + } + +} + +impl<'a> Default for Serialized<'a> { + fn default() -> Self { + Serialized(Cow::Borrowed(&[0u8; 4][..])) + } +} + +impl<'a> Serialized<'a> { + + // Index at end, also contains the encoded size + fn index_start(&self) -> usize { + let nb_ix = self.len(); + let end = self.0.len(); + end - (nb_ix * 4) + } + + fn index_element(&self, position: usize) -> usize { + let index_start = self.index_start(); + if position == 0 { + return index_start; + } + let i = index_start + (position - 1) * 4; + self.read_le_usize(i) + } + + // move part of array that can overlap + // This is a memory inefficient implementation. + fn slice_copy(&mut self, start_from: usize, start_to: usize, size: usize) { + let buffer = self.0[start_from..start_from + size].to_vec(); + self.0.to_mut()[start_to..start_to + size].copy_from_slice(&buffer[..]); + } + + // Usize encoded as le u32. + fn read_le_usize(&self, pos: usize) -> usize { + let mut buffer = [0u8; 4]; + buffer.copy_from_slice(&self.0[pos..pos + 4]); + u32::from_le_bytes(buffer) as usize + } + + // Usize encoded as le u32. + fn write_le_usize(&mut self, pos: usize, value: usize) { + let buffer = (value as u32).to_le_bytes(); + self.0.to_mut()[pos..pos + 4].copy_from_slice(&buffer[..]); + } + + // Usize encoded as le u32. + fn append_le_usize(&mut self, value: usize) { + let buffer = (value as u32).to_le_bytes(); + self.0.to_mut().extend_from_slice(&buffer[..]); + } + +} + +impl<'a> Serialized<'a> { + + /// Set a value, it uses a state history as parameter. + /// This method uses `get_mut` and do remove pending + /// dropped value. + pub fn set(&mut self, history: &[TransactionState], val: &[u8]) { + if let Some((_v, index)) = self.get_mut(history) { + if index == history.len() - 1 { + self.pop(); + self.push((val, index)); + return; + } + } + self.push((val, history.len() - 1)); + } + + fn mut_ref(&mut self, _index: usize) -> () { + () + } + +} + + +// TODO macroed this big copy (and maybe later spend time on using a trait: not +// worth it at this point). +impl<'a> Serialized<'a> { + + /// Access to latest pending value (non dropped state in history). + /// When possible please prefer `get_mut` as it can free + /// some memory. + pub fn get(&self, history: &[TransactionState]) -> Option<&[u8]> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let (v, history_index) = self.get_state(index); + match history[history_index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Committed => + return Some(v), + } + } + None + } + + #[cfg(any(test, feature = "test"))] + pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&[u8]> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let (v, history_index) = self.get_state(index); + match history[history_index] { + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => + return Some(v), + TransactionState::Committed + | TransactionState::Dropped => (), + } + } + None + } + + #[cfg(any(test, feature = "test"))] + pub fn get_committed(&self, history: &[TransactionState]) -> Option<&[u8]> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let (v, history_index) = self.get_state(index); + match history[history_index] { + TransactionState::Committed => + return Some(v), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + pub fn into_committed(mut self, history: &[TransactionState]) -> Option> { + // index is never 0, + let mut index = self.len(); + if index == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).1; + match history[history_index] { + TransactionState::Committed => { + self.truncate(index + 1); + return self.pop().map(|v| v.0); + }, + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Dropped => (), + } + } + None + } + + /// Access to latest pending value (non dropped state in history). + /// This method uses `get_mut` and do remove pending + /// This method remove latest dropped value up to the latest valid value. + pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<((), usize)> { + + let mut index = self.len(); + if index == 0 { + return None; + } + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).1; + match history[history_index] { + TransactionState::Committed => { + // here we could gc all preceding values but that is additional cost + // and get_mut should stop at pending following committed. + return Some((self.mut_ref(index), history_index)) + }, + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective => { + return Some((self.mut_ref(index), history_index)) + }, + TransactionState::Dropped => { let _ = self.pop(); }, + } + } + None + } + + + /// Garbage collect a history, act as a `get_mut` with additional cost. + /// If `eager` is true, all dropped value are removed even if it means shifting + /// array byte. Otherwhise we mainly ensure collection up to last Commit state + /// (truncate left size). + pub fn gc( + &mut self, + history: &[TransactionState], + tx_index: Option<&[usize]>, + ) -> Option<((), usize)> { + if let Some(tx_index) = tx_index { + let mut tx_index = &tx_index[..]; + let mut index = self.len(); + if index == 0 { + return None; + } + let mut bellow_value = usize::max_value(); + let mut result: Option<(usize, usize)> = None; + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).1; + match history[history_index] { + TransactionState::Committed => { + for _ in 0..index { + self.remove(0); + } + result = Some(result.map(|(i, hi)| (i - index, hi)).unwrap_or((0, history_index))); + index = 0; + }, + TransactionState::Pending + | TransactionState::Prospective => { + if history_index >= bellow_value { + self.remove(index); + result.as_mut().map(|(i, _hi)| *i = *i - 1); + } else { + if result.is_none() { + result = Some((index, history_index)); + } + while bellow_value > history_index { + // bellow_value = pop + let split = tx_index.split_last() + .map(|(v, sl)| (*v, sl)) + .unwrap_or((0, &[])); + bellow_value = split.0; + tx_index = split.1; + } + } + }, + TransactionState::TxPending => { + if history_index >= bellow_value { + self.remove(index); + result.as_mut().map(|(i, _hi)| *i = *i - 1); + } else { + if result.is_none() { + result = Some((index, history_index)); + } + } + bellow_value = usize::max_value(); + }, + TransactionState::Dropped => { + self.remove(index); + }, + } + } + if let Some((index, history_index)) = result { + Some((self.mut_ref(index), history_index)) + } else { None } + + } else { + return self.get_mut(history); + } + } +} + + diff --git a/core/utils/history-driven-data/src/linear.rs b/core/utils/history-driven-data/src/linear.rs deleted file mode 100644 index 0896b4a822458..0000000000000 --- a/core/utils/history-driven-data/src/linear.rs +++ /dev/null @@ -1,480 +0,0 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Transactional overlay implementation. -//! -//! This follows a linear succession of states. -//! This contains multiple unbounded transaction layer -//! and an additional top level 'prospective' layer. -//! It only allows linear history (no branch so -//! inner storage is only an array of element). - -use crate::State as TransactionState; -use rstd::vec::Vec; -use rstd::vec; -use rstd::marker::PhantomData; - -/// Array like buffer for in memory storage. -/// By in memory we expect that this will -/// not required persistence and is not serialized. -#[cfg(not(feature = "std"))] -pub type MemoryOnly = Vec<(V, usize)>; - -/// Array like buffer for in memory storage. -/// By in memory we expect that this will -/// not required persistence and is not serialized. -#[cfg(feature = "std")] -pub type MemoryOnly = smallvec::SmallVec<[(V, usize); ALLOCATED_HISTORY]>; - -/// Size of preallocated history per element. -/// Currently at two for committed and prospective only. -/// It means that using transaction in a module got a direct allocation cost. -const ALLOCATED_HISTORY: usize = 2; - -/// History of value that are related to a state history (eg field `history` of -/// an `OverlayedChangeSet`). -/// -/// Values are always paired with a state history index. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] -pub struct History(B, PhantomData); - -trait HistoryBuffer: Default - + rstd::ops::Index - + rstd::ops::IndexMut -{ - - fn len(&self) -> usize; - fn truncate(&mut self, _index: usize); - fn pop(&mut self) -> Option<(V, usize)>; - fn push(&mut self, _val: (V, usize)); - fn remove(&mut self, _index: usize) -> (V, usize); -} - -impl HistoryBuffer for MemoryOnly { - fn len(&self) -> usize { - self.len() - } - - fn truncate(&mut self, index: usize) { - self.truncate(index) - } - - fn pop(&mut self) -> Option<(V, usize)> { - self.pop() - } - - fn push(&mut self, val: (V, usize)) { - self.push(val) - } - - fn remove(&mut self, index: usize) -> (V, usize) { - self.remove(index) - } - - - -} - -impl, V> Default for History { - fn default() -> Self { - History(Default::default(), PhantomData) - } -} - -impl, V> rstd::ops::Index for History { - type Output = (V, usize); - fn index(&self, index: usize) -> &Self::Output { - self.0.index(index) - } -} - -impl, V> rstd::ops::IndexMut for History { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - self.0.index_mut(index) - } -} - -impl, V> History { - - #[cfg(any(test, feature = "test"))] - /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator) -> Self { - let mut history = History::default(); - for v in input { - history.force_push(v.0, v.1); - } - history - } - - #[cfg(any(test, feature = "test"))] - /// Debugging function for test and fuzzing. - pub fn internal_item_counts(&self) -> usize { - self.0.len() - } - - fn len(&self) -> usize { - self.0.len() - } - - fn truncate(&mut self, index: usize) { - self.0.truncate(index) - } - - fn pop(&mut self) -> Option<(V, usize)> { - self.0.pop() - } - - /// Append without checking if a value already exist. - /// If a value already exists, the history will be broken. - /// This method shall only be call after a `get_mut` where - /// the returned index indicate that a `set` will result - /// in appending a value. - pub fn force_push(&mut self, val: V, tx_index: usize) { - self.0.push((val, tx_index)) - } - - /// Access to latest pending value (non dropped state in history). - /// When possible please prefer `get_mut` as it can free - /// some memory. - pub fn get(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Committed => - return Some(&self[index].0), - } - } - None - } - - #[cfg(any(test, feature = "test"))] - pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => - return Some(&self[index].0), - TransactionState::Committed - | TransactionState::Dropped => (), - } - } - None - } - - #[cfg(any(test, feature = "test"))] - pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => - return Some(&self[index].0), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), - } - } - None - } - - pub fn into_committed(mut self, history: &[TransactionState]) -> Option { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => { - self.truncate(index + 1); - return self.pop().map(|v| v.0); - }, - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), - } - } - None - } - - /// Access to latest pending value (non dropped state in history). - /// This method uses `get_mut` and do remove pending - /// This method remove latest dropped value up to the latest valid value. - pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { - - let mut index = self.len(); - if index == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => { - // here we could gc all preceding values but that is additional cost - // and get_mut should stop at pending following committed. - let tx = self[index].1; - return Some((&mut self[index].0, tx)) - }, - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => { - let tx = self[index].1; - return Some((&mut self[index].0, tx)) - }, - TransactionState::Dropped => { let _ = self.pop(); }, - } - } - None - } - - /// Set a value, it uses a state history as parameter. - /// This method uses `get_mut` and do remove pending - /// dropped value. - pub fn set(&mut self, history: &[TransactionState], val: V) { - if let Some((v, index)) = self.get_mut(history) { - if index == history.len() - 1 { - *v = val; - return; - } - } - self.force_push(val, history.len() - 1); - } - - /// Garbage collect a history, act as a `get_mut` with additional cost. - /// If `eager` is true, all dropped value are removed even if it means shifting - /// array byte. Otherwhise we mainly ensure collection up to last Commit state - /// (truncate left size). - pub fn gc( - &mut self, - history: &[TransactionState], - tx_index: Option<&[usize]>, - ) -> Option<(&mut V, usize)> { - if let Some(tx_index) = tx_index { - let mut tx_index = &tx_index[..]; - let mut index = self.len(); - if index == 0 { - return None; - } - let mut bellow_value = usize::max_value(); - let mut result: Option = None; - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self[index].1; - match history[history_index] { - TransactionState::Committed => { - for _ in 0..index { - let _ = self.0.remove(0); - } - result = Some(result.map(|i| i - index).unwrap_or(0)); - index = 0; - }, - TransactionState::Pending - | TransactionState::Prospective => { - if history_index >= bellow_value { - let _ = self.0.remove(index); - result.as_mut().map(|i| *i = *i - 1); - } else { - if result.is_none() { - result = Some(index); - } - while bellow_value > history_index { - // bellow_value = pop - let split = tx_index.split_last() - .map(|(v, sl)| (*v, sl)) - .unwrap_or((0, &[])); - bellow_value = split.0; - tx_index = split.1; - } - } - }, - TransactionState::TxPending => { - if history_index >= bellow_value { - let _ = self.0.remove(index); - result.as_mut().map(|i| *i = *i - 1); - } else { - if result.is_none() { - result = Some(index); - } - } - bellow_value = usize::max_value(); - }, - TransactionState::Dropped => { - let _ = self.0.remove(index); - }, - } - } - if let Some(index) = result { - let tx = self[index].1.clone(); - Some((&mut self[index].0, tx)) - } else { None } - - } else { - return self.get_mut(history); - } - } -} - - -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] -pub struct States(Vec); - -impl Default for States { - fn default() -> Self { - States(vec![TransactionState::Pending]) - } -} - -impl States { - pub fn as_ref(&self) -> &[TransactionState] { - self.0.as_ref() - } - - pub fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.0.iter().map(Clone::clone).enumerate() - } -} - -impl States { - - /// Build any state for testing only. - #[cfg(any(test, feature = "test"))] - pub fn test_vector(test_state: Vec) -> Self { - States(test_state) - } - - /// Discard prospective changes to state. - pub fn discard_prospective(&mut self) { - let mut i = self.0.len(); - while i > 0 { - i -= 1; - match self.0[i] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => self.0[i] = TransactionState::Dropped, - TransactionState::Committed => break, - } - } - self.0.push(TransactionState::Pending); - } - - /// Commit prospective changes to state. - pub fn commit_prospective(&mut self) { - debug_assert!(self.0.len() > 0); - let mut i = self.0.len(); - while i > 0 { - i -= 1; - match self.0[i] { - TransactionState::Dropped => (), - TransactionState::Prospective - | TransactionState::TxPending - | TransactionState::Pending => self.0[i] = TransactionState::Committed, - | TransactionState::Committed => break, - } - } - self.0.push(TransactionState::Pending); - } - - /// Create a new transactional layer. - pub fn start_transaction(&mut self) { - self.0.push(TransactionState::TxPending); - } - - /// Discard a transactional layer. - /// A transaction is always running (history always end with pending). - pub fn discard_transaction(&mut self) { - let mut i = self.0.len(); - while i > 0 { - i -= 1; - match self.0[i] { - TransactionState::Dropped => (), - TransactionState::Prospective - | TransactionState::Pending => self.0[i] = TransactionState::Dropped, - TransactionState::TxPending => { - self.0[i] = TransactionState::Dropped; - break; - }, - TransactionState::Committed => break, - } - } - self.0.push(TransactionState::Pending); - } - - /// Commit a transactional layer. - pub fn commit_transaction(&mut self) { - let mut i = self.0.len(); - while i > 0 { - i -= 1; - match self.0[i] { - TransactionState::Prospective - | TransactionState::Dropped => (), - TransactionState::Pending => self.0[i] = TransactionState::Prospective, - TransactionState::TxPending => { - self.0[i] = TransactionState::Prospective; - break; - }, - TransactionState::Committed => break, - } - } - self.0.push(TransactionState::Pending); - } - -} From 46de531c45df13b7d8469af62c80f246bbd988ad Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 28 Aug 2019 09:54:25 +0200 Subject: [PATCH 030/129] macroed code (trait would be better but not at this point). --- core/utils/historied-data/src/linear.rs | 241 ++---------------------- 1 file changed, 18 insertions(+), 223 deletions(-) diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index a4af2574eef3c..267a842ee2658 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -65,218 +65,6 @@ impl Default for History { } } -impl History { - - /// Access to latest pending value (non dropped state in history). - /// When possible please prefer `get_mut` as it can free - /// some memory. - pub fn get(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let (v, history_index) = self.get_state(index); - match history[history_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Committed => - return Some(v), - } - } - None - } - - #[cfg(any(test, feature = "test"))] - pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let (v, history_index) = self.get_state(index); - match history[history_index] { - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => - return Some(v), - TransactionState::Committed - | TransactionState::Dropped => (), - } - } - None - } - - #[cfg(any(test, feature = "test"))] - pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let (v, history_index) = self.get_state(index); - match history[history_index] { - TransactionState::Committed => - return Some(v), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), - } - } - None - } - - pub fn into_committed(mut self, history: &[TransactionState]) -> Option { - // index is never 0, - let mut index = self.len(); - if index == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self.get_state(index).1; - match history[history_index] { - TransactionState::Committed => { - self.truncate(index + 1); - return self.pop().map(|v| v.0); - }, - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), - } - } - None - } - - /// Access to latest pending value (non dropped state in history). - /// This method uses `get_mut` and do remove pending - /// This method remove latest dropped value up to the latest valid value. - pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { - - let mut index = self.len(); - if index == 0 { - return None; - } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self.get_state(index).1; - match history[history_index] { - TransactionState::Committed => { - // here we could gc all preceding values but that is additional cost - // and get_mut should stop at pending following committed. - return Some((self.mut_ref(index), history_index)) - }, - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => { - return Some((self.mut_ref(index), history_index)) - }, - TransactionState::Dropped => { let _ = self.pop(); }, - } - } - None - } - - - /// Garbage collect a history, act as a `get_mut` with additional cost. - /// If `eager` is true, all dropped value are removed even if it means shifting - /// array byte. Otherwhise we mainly ensure collection up to last Commit state - /// (truncate left size). - pub fn gc( - &mut self, - history: &[TransactionState], - tx_index: Option<&[usize]>, - ) -> Option<(&mut V, usize)> { - if let Some(tx_index) = tx_index { - let mut tx_index = &tx_index[..]; - let mut index = self.len(); - if index == 0 { - return None; - } - let mut bellow_value = usize::max_value(); - let mut result: Option<(usize, usize)> = None; - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self.get_state(index).1; - match history[history_index] { - TransactionState::Committed => { - for _ in 0..index { - self.remove(0); - } - result = Some(result.map(|(i, hi)| (i - index, hi)).unwrap_or((0, history_index))); - index = 0; - }, - TransactionState::Pending - | TransactionState::Prospective => { - if history_index >= bellow_value { - self.remove(index); - result.as_mut().map(|(i, _hi)| *i = *i - 1); - } else { - if result.is_none() { - result = Some((index, history_index)); - } - while bellow_value > history_index { - // bellow_value = pop - let split = tx_index.split_last() - .map(|(v, sl)| (*v, sl)) - .unwrap_or((0, &[])); - bellow_value = split.0; - tx_index = split.1; - } - } - }, - TransactionState::TxPending => { - if history_index >= bellow_value { - self.remove(index); - result.as_mut().map(|(i, _hi)| *i = *i - 1); - } else { - if result.is_none() { - result = Some((index, history_index)); - } - } - bellow_value = usize::max_value(); - }, - TransactionState::Dropped => { - self.remove(index); - }, - } - } - if let Some((index, history_index)) = result { - Some((self.mut_ref(index), history_index)) - } else { None } - - } else { - return self.get_mut(history); - } - } -} - - impl History { #[cfg(any(test, feature = "test"))] @@ -622,14 +410,14 @@ impl<'a> Serialized<'a> { } -// TODO macroed this big copy (and maybe later spend time on using a trait: not -// worth it at this point). -impl<'a> Serialized<'a> { + +// share implementation, trait would be better. +macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { /// Access to latest pending value (non dropped state in history). /// When possible please prefer `get_mut` as it can free /// some memory. - pub fn get(&self, history: &[TransactionState]) -> Option<&[u8]> { + pub fn get(&self, history: &[TransactionState]) -> Option<$read> { // index is never 0, let mut index = self.len(); if index == 0 { @@ -652,7 +440,7 @@ impl<'a> Serialized<'a> { } #[cfg(any(test, feature = "test"))] - pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&[u8]> { + pub fn get_prospective(&self, history: &[TransactionState]) -> Option<$read> { // index is never 0, let mut index = self.len(); if index == 0 { @@ -675,7 +463,7 @@ impl<'a> Serialized<'a> { } #[cfg(any(test, feature = "test"))] - pub fn get_committed(&self, history: &[TransactionState]) -> Option<&[u8]> { + pub fn get_committed(&self, history: &[TransactionState]) -> Option<$read> { // index is never 0, let mut index = self.len(); if index == 0 { @@ -697,7 +485,7 @@ impl<'a> Serialized<'a> { None } - pub fn into_committed(mut self, history: &[TransactionState]) -> Option> { + pub fn into_committed(mut self, history: &[TransactionState]) -> Option<$owned> { // index is never 0, let mut index = self.len(); if index == 0 { @@ -725,9 +513,9 @@ impl<'a> Serialized<'a> { } /// Access to latest pending value (non dropped state in history). - /// This method uses `get_mut` and do remove pending + /// /// This method remove latest dropped value up to the latest valid value. - pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<((), usize)> { + pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<($mut, usize)> { let mut index = self.len(); if index == 0 { @@ -766,7 +554,7 @@ impl<'a> Serialized<'a> { &mut self, history: &[TransactionState], tx_index: Option<&[usize]>, - ) -> Option<((), usize)> { + ) -> Option<($mut, usize)> { if let Some(tx_index) = tx_index { let mut tx_index = &tx_index[..]; let mut index = self.len(); @@ -833,6 +621,13 @@ impl<'a> Serialized<'a> { return self.get_mut(history); } } -} +}); + +impl<'a> Serialized<'a> { + history_impl!(&[u8], Vec, ()); +} +impl History { + history_impl!(&V, V, &mut V); +} From 80f4c2bb407fedbfb67d495b35317c8c4bf09928 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 28 Aug 2019 11:33:48 +0200 Subject: [PATCH 031/129] Fix basis code. --- core/utils/historied-data/src/linear.rs | 129 +++++++++++++++++++----- 1 file changed, 104 insertions(+), 25 deletions(-) diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 267a842ee2658..3dcaf56214d66 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -243,6 +243,10 @@ impl States { /// Could be use for direct access memory to. pub struct Serialized<'a>(Cow<'a, [u8]>); +// Basis implementation to be on par with implementation using +// vec like container. Those method could be move to a trait +// implementation. +// Those function requires checked index. impl<'a> Serialized<'a> { fn len(&self) -> usize { @@ -250,17 +254,26 @@ impl<'a> Serialized<'a> { self.read_le_usize(len - 4) } + fn clear(&mut self) { + self.write_le_usize(0, 0); + self.0.to_mut().truncate(4); + } + fn truncate(&mut self, index: usize) { + if index == 0 { + self.clear(); + return; + } let len = self.len(); if index >= len { return; } - let start = self.index_start(); - let end_ix = self.index_element(index + 1); - let len_ix = len * 4; - self.slice_copy(start, end_ix, len_ix); - self.write_le_usize(end_ix + len_ix - 4, index); - self.0.to_mut().truncate(end_ix + len_ix); + let start_ix = self.index_start(); + let new_start = self.index_element(index); + let len_ix = index * 4; + self.slice_copy(start_ix, new_start, len_ix); + self.write_le_usize(new_start + len_ix - 4, index); + self.0.to_mut().truncate(new_start + len_ix); } fn pop(&mut self) -> Option<(Vec, usize)> { @@ -268,12 +281,17 @@ impl<'a> Serialized<'a> { if len == 0 { return None; } - let start_ix = self.index_element(len); + let start_ix = self.index_element(len - 1); let end_ix = self.index_start(); let state = self.read_le_usize(start_ix); let value = self.0[start_ix + 4..end_ix].to_vec(); - self.write_le_usize(len - 1, self.0.len() - 8); - let ix_size = len * 4; + if len - 1 == 0 { + self.clear(); + return Some((value, state)) + } else { + self.write_le_usize(self.0.len() - 8, len - 1); + }; + let ix_size = (len * 4) - 4; self.slice_copy(end_ix, start_ix, ix_size); self.0.to_mut().truncate(start_ix + ix_size); Some((value, state)) @@ -291,33 +309,42 @@ impl<'a> Serialized<'a> { self.write_le_usize(start_ix, val.1); self.0.to_mut().extend_from_slice(val.0); self.0.to_mut().append(&mut new_ix); - self.write_le_usize(self.0.len() - 4, start_ix); - self.append_le_usize(len + 1); + if len > 0 { + self.write_le_usize(self.0.len() - 4, start_ix); + self.append_le_usize(len + 1); + } else { + self.write_le_usize(self.0.len() - 4, 1); + } } fn remove(&mut self, index: usize) { let len = self.len(); + if len == 1 && index == 0 { + self.clear(); + return; + } // eager removal is costy, running some gc impl // can be interesting (would be malleable serializing). let elt_start = self.index_element(index); let start_ix = self.index_start(); - let elt_end = if index == len { + let elt_end = if index == len - 1 { start_ix } else { - self.index_element(index) + self.index_element(index + 1) }; let delete_size = elt_end - elt_start; - for i in elt_start..elt_end { - let _ = self.0.to_mut().remove(i); + for _ in elt_start..elt_end { + let _ = self.0.to_mut().remove(elt_start); } let start_ix = start_ix - delete_size; - for i in 0..len - index - 1 { - let old_value = self.read_le_usize(4 + start_ix + i * 4); - self.write_le_usize(start_ix + i * 4, old_value - delete_size); + for i in 1..len - index - 1 { + let old_value = self.read_le_usize(start_ix + i * 4); + self.write_le_usize(start_ix + (i - 1) * 4, old_value - delete_size); } - let size_index = start_ix + (len - 1) * 4; - self.write_le_usize(size_index, len - 1); - self.0.to_mut().truncate(size_index + 4); + let len = len - 1; + let end_index = start_ix + len * 4; + self.write_le_usize(end_index - 4, len); + self.0.to_mut().truncate(end_index); } @@ -330,7 +357,7 @@ impl<'a> Serialized<'a> { self.index_element(index + 1) }; let state = self.read_le_usize(start_ix); - (&self.0[start_ix..end_ix], state) + (&self.0[start_ix + 4..end_ix], state) } } @@ -341,21 +368,22 @@ impl<'a> Default for Serialized<'a> { } } +// Utility function for basis implementation. impl<'a> Serialized<'a> { // Index at end, also contains the encoded size fn index_start(&self) -> usize { let nb_ix = self.len(); + if nb_ix ==0 { return 0; } let end = self.0.len(); end - (nb_ix * 4) } fn index_element(&self, position: usize) -> usize { - let index_start = self.index_start(); if position == 0 { - return index_start; + return 0; } - let i = index_start + (position - 1) * 4; + let i = self.index_start() + (position - 1) * 4; self.read_le_usize(i) } @@ -631,3 +659,54 @@ impl<'a> Serialized<'a> { impl History { history_impl!(&V, V, &mut V); } + +mod test { + use super::*; + + #[test] + fn serialized_basis() { + // test basis unsafe function similar to a simple vec + // without index checking. + //fn get_state(&self, index: usize) -> (&V, usize) { + //fn len(&self) -> usize { + //fn truncate(&mut self, index: usize) { + //fn push(&mut self, val: (&[u8], usize)) { + //fn pop(&mut self) -> Option<(Vec, usize)> { + //fn remove(&mut self, index: usize) { + //fn get_state(&self, index: usize) -> (&[u8], usize) { + let v1 = &b"val1"[..]; + let v2 = &b"value_2"[..]; + let v3 = &b"a third value 3"[..]; + let mut ser = Serialized::default(); + assert_eq!(ser.len(), 0); + assert_eq!(ser.pop(), None); + ser.push((v1, 1)); + assert_eq!(ser.get_state(0), (v1, 1)); + assert_eq!(ser.pop(), Some((v1.to_vec(), 1))); + assert_eq!(ser.len(), 0); + ser.push((v1, 1)); + ser.push((v2, 2)); + ser.push((v3, 3)); + assert_eq!(ser.get_state(0), (v1, 1)); + assert_eq!(ser.get_state(1), (v2, 2)); + assert_eq!(ser.get_state(2), (v3, 3)); + assert_eq!(ser.pop(), Some((v3.to_vec(), 3))); + assert_eq!(ser.len(), 2); + ser.push((v3, 3)); + assert_eq!(ser.get_state(2), (v3, 3)); + ser.remove(0); + assert_eq!(ser.len(), 2); + assert_eq!(ser.get_state(0), (v2, 2)); + assert_eq!(ser.get_state(1), (v3, 3)); + ser.push((v1, 1)); + ser.remove(1); + assert_eq!(ser.len(), 2); + assert_eq!(ser.get_state(0), (v2, 2)); + assert_eq!(ser.get_state(1), (v1, 1)); + ser.push((v1, 1)); + ser.truncate(1); + assert_eq!(ser.len(), 1); + assert_eq!(ser.get_state(0), (v2, 2)); + + } +} From 8dbd48d859f1463f13744af8a04f68793ff8f1a5 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 28 Aug 2019 11:38:00 +0200 Subject: [PATCH 032/129] Remove serialized variant. --- core/utils/historied-data/src/linear.rs | 279 +----------------------- 1 file changed, 7 insertions(+), 272 deletions(-) diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 3dcaf56214d66..07ae1188c7b11 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -25,7 +25,6 @@ use crate::State as TransactionState; use rstd::vec::Vec; use rstd::vec; -use rstd::borrow::Cow; /// Array like buffer for in memory storage. /// By in memory we expect that this will @@ -234,218 +233,12 @@ impl States { } - -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] -/// Arraylike buffer with in place byte data. -/// Can be written as is in underlying -/// storage. -/// Could be use for direct access memory to. -pub struct Serialized<'a>(Cow<'a, [u8]>); - -// Basis implementation to be on par with implementation using -// vec like container. Those method could be move to a trait -// implementation. -// Those function requires checked index. -impl<'a> Serialized<'a> { - - fn len(&self) -> usize { - let len = self.0.len(); - self.read_le_usize(len - 4) - } - - fn clear(&mut self) { - self.write_le_usize(0, 0); - self.0.to_mut().truncate(4); - } - - fn truncate(&mut self, index: usize) { - if index == 0 { - self.clear(); - return; - } - let len = self.len(); - if index >= len { - return; - } - let start_ix = self.index_start(); - let new_start = self.index_element(index); - let len_ix = index * 4; - self.slice_copy(start_ix, new_start, len_ix); - self.write_le_usize(new_start + len_ix - 4, index); - self.0.to_mut().truncate(new_start + len_ix); - } - - fn pop(&mut self) -> Option<(Vec, usize)> { - let len = self.len(); - if len == 0 { - return None; - } - let start_ix = self.index_element(len - 1); - let end_ix = self.index_start(); - let state = self.read_le_usize(start_ix); - let value = self.0[start_ix + 4..end_ix].to_vec(); - if len - 1 == 0 { - self.clear(); - return Some((value, state)) - } else { - self.write_le_usize(self.0.len() - 8, len - 1); - }; - let ix_size = (len * 4) - 4; - self.slice_copy(end_ix, start_ix, ix_size); - self.0.to_mut().truncate(start_ix + ix_size); - Some((value, state)) - } - - fn push(&mut self, val: (&[u8], usize)) { - let len = self.len(); - let start_ix = self.index_start(); - let end_ix = self.0.len(); - // A sized buffer and multiple index to avoid to big copy - // should be use here. - let mut new_ix = self.0[start_ix..end_ix].to_vec(); - // truncate here can be bad - self.0.to_mut().truncate(start_ix + 4); - self.write_le_usize(start_ix, val.1); - self.0.to_mut().extend_from_slice(val.0); - self.0.to_mut().append(&mut new_ix); - if len > 0 { - self.write_le_usize(self.0.len() - 4, start_ix); - self.append_le_usize(len + 1); - } else { - self.write_le_usize(self.0.len() - 4, 1); - } - } - - fn remove(&mut self, index: usize) { - let len = self.len(); - if len == 1 && index == 0 { - self.clear(); - return; - } - // eager removal is costy, running some gc impl - // can be interesting (would be malleable serializing). - let elt_start = self.index_element(index); - let start_ix = self.index_start(); - let elt_end = if index == len - 1 { - start_ix - } else { - self.index_element(index + 1) - }; - let delete_size = elt_end - elt_start; - for _ in elt_start..elt_end { - let _ = self.0.to_mut().remove(elt_start); - } - let start_ix = start_ix - delete_size; - for i in 1..len - index - 1 { - let old_value = self.read_le_usize(start_ix + i * 4); - self.write_le_usize(start_ix + (i - 1) * 4, old_value - delete_size); - } - let len = len - 1; - let end_index = start_ix + len * 4; - self.write_le_usize(end_index - 4, len); - self.0.to_mut().truncate(end_index); - - } - - fn get_state(&self, index: usize) -> (&[u8], usize) { - let start_ix = self.index_element(index); - let len = self.len(); - let end_ix = if index == len - 1 { - self.index_start() - } else { - self.index_element(index + 1) - }; - let state = self.read_le_usize(start_ix); - (&self.0[start_ix + 4..end_ix], state) - } - -} - -impl<'a> Default for Serialized<'a> { - fn default() -> Self { - Serialized(Cow::Borrowed(&[0u8; 4][..])) - } -} - -// Utility function for basis implementation. -impl<'a> Serialized<'a> { - - // Index at end, also contains the encoded size - fn index_start(&self) -> usize { - let nb_ix = self.len(); - if nb_ix ==0 { return 0; } - let end = self.0.len(); - end - (nb_ix * 4) - } - - fn index_element(&self, position: usize) -> usize { - if position == 0 { - return 0; - } - let i = self.index_start() + (position - 1) * 4; - self.read_le_usize(i) - } - - // move part of array that can overlap - // This is a memory inefficient implementation. - fn slice_copy(&mut self, start_from: usize, start_to: usize, size: usize) { - let buffer = self.0[start_from..start_from + size].to_vec(); - self.0.to_mut()[start_to..start_to + size].copy_from_slice(&buffer[..]); - } - - // Usize encoded as le u32. - fn read_le_usize(&self, pos: usize) -> usize { - let mut buffer = [0u8; 4]; - buffer.copy_from_slice(&self.0[pos..pos + 4]); - u32::from_le_bytes(buffer) as usize - } - - // Usize encoded as le u32. - fn write_le_usize(&mut self, pos: usize, value: usize) { - let buffer = (value as u32).to_le_bytes(); - self.0.to_mut()[pos..pos + 4].copy_from_slice(&buffer[..]); - } - - // Usize encoded as le u32. - fn append_le_usize(&mut self, value: usize) { - let buffer = (value as u32).to_le_bytes(); - self.0.to_mut().extend_from_slice(&buffer[..]); - } - -} - -impl<'a> Serialized<'a> { - - /// Set a value, it uses a state history as parameter. - /// This method uses `get_mut` and do remove pending - /// dropped value. - pub fn set(&mut self, history: &[TransactionState], val: &[u8]) { - if let Some((_v, index)) = self.get_mut(history) { - if index == history.len() - 1 { - self.pop(); - self.push((val, index)); - return; - } - } - self.push((val, history.len() - 1)); - } - - fn mut_ref(&mut self, _index: usize) -> () { - () - } - -} - - - -// share implementation, trait would be better. -macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { +impl History { /// Access to latest pending value (non dropped state in history). /// When possible please prefer `get_mut` as it can free /// some memory. - pub fn get(&self, history: &[TransactionState]) -> Option<$read> { + pub fn get(&self, history: &[TransactionState]) -> Option<&V> { // index is never 0, let mut index = self.len(); if index == 0 { @@ -468,7 +261,7 @@ macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { } #[cfg(any(test, feature = "test"))] - pub fn get_prospective(&self, history: &[TransactionState]) -> Option<$read> { + pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { // index is never 0, let mut index = self.len(); if index == 0 { @@ -491,7 +284,7 @@ macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { } #[cfg(any(test, feature = "test"))] - pub fn get_committed(&self, history: &[TransactionState]) -> Option<$read> { + pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { // index is never 0, let mut index = self.len(); if index == 0 { @@ -513,7 +306,7 @@ macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { None } - pub fn into_committed(mut self, history: &[TransactionState]) -> Option<$owned> { + pub fn into_committed(mut self, history: &[TransactionState]) -> Option { // index is never 0, let mut index = self.len(); if index == 0 { @@ -543,7 +336,7 @@ macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { /// Access to latest pending value (non dropped state in history). /// /// This method remove latest dropped value up to the latest valid value. - pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<($mut, usize)> { + pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { let mut index = self.len(); if index == 0 { @@ -582,7 +375,7 @@ macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { &mut self, history: &[TransactionState], tx_index: Option<&[usize]>, - ) -> Option<($mut, usize)> { + ) -> Option<(&mut V, usize)> { if let Some(tx_index) = tx_index { let mut tx_index = &tx_index[..]; let mut index = self.len(); @@ -649,64 +442,6 @@ macro_rules! history_impl(( $read: ty, $owned: ty, $mut: ty ) => { return self.get_mut(history); } } - -}); - -impl<'a> Serialized<'a> { - history_impl!(&[u8], Vec, ()); } -impl History { - history_impl!(&V, V, &mut V); -} -mod test { - use super::*; - - #[test] - fn serialized_basis() { - // test basis unsafe function similar to a simple vec - // without index checking. - //fn get_state(&self, index: usize) -> (&V, usize) { - //fn len(&self) -> usize { - //fn truncate(&mut self, index: usize) { - //fn push(&mut self, val: (&[u8], usize)) { - //fn pop(&mut self) -> Option<(Vec, usize)> { - //fn remove(&mut self, index: usize) { - //fn get_state(&self, index: usize) -> (&[u8], usize) { - let v1 = &b"val1"[..]; - let v2 = &b"value_2"[..]; - let v3 = &b"a third value 3"[..]; - let mut ser = Serialized::default(); - assert_eq!(ser.len(), 0); - assert_eq!(ser.pop(), None); - ser.push((v1, 1)); - assert_eq!(ser.get_state(0), (v1, 1)); - assert_eq!(ser.pop(), Some((v1.to_vec(), 1))); - assert_eq!(ser.len(), 0); - ser.push((v1, 1)); - ser.push((v2, 2)); - ser.push((v3, 3)); - assert_eq!(ser.get_state(0), (v1, 1)); - assert_eq!(ser.get_state(1), (v2, 2)); - assert_eq!(ser.get_state(2), (v3, 3)); - assert_eq!(ser.pop(), Some((v3.to_vec(), 3))); - assert_eq!(ser.len(), 2); - ser.push((v3, 3)); - assert_eq!(ser.get_state(2), (v3, 3)); - ser.remove(0); - assert_eq!(ser.len(), 2); - assert_eq!(ser.get_state(0), (v2, 2)); - assert_eq!(ser.get_state(1), (v3, 3)); - ser.push((v1, 1)); - ser.remove(1); - assert_eq!(ser.len(), 2); - assert_eq!(ser.get_state(0), (v2, 2)); - assert_eq!(ser.get_state(1), (v1, 1)); - ser.push((v1, 1)); - ser.truncate(1); - assert_eq!(ser.len(), 1); - assert_eq!(ser.get_state(0), (v2, 2)); - - } -} From 9203e25ab2b102790d7426f48e9955691b6f4c24 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 28 Aug 2019 12:24:47 +0200 Subject: [PATCH 033/129] change some variables name, try to document gc. --- core/state-machine/src/overlayed_changes.rs | 89 ++++++++++++--------- core/utils/historied-data/src/linear.rs | 59 +++++++------- 2 files changed, 80 insertions(+), 68 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 0c5dc80fa4a66..cad7235b4c4c2 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -86,37 +86,35 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { } } - - /// Variant of `set` value that update extrinsics. /// It does remove latest dropped values. fn set_with_extrinsic_overlayed_value( - h: &mut History, + h_value: &mut History, history: &[TransactionState], - val: Option>, + value: Option>, extrinsic_index: Option, ) { if let Some(extrinsic) = extrinsic_index { - set_with_extrinsic_inner_overlayed_value(h, history, val, extrinsic) + set_with_extrinsic_inner_overlayed_value(h_value, history, value, extrinsic) } else { - h.set(history, OverlayedValue { - value: val, + h_value.set(history, OverlayedValue { + value, extrinsics: None, }) } } fn set_with_extrinsic_inner_overlayed_value( - h: &mut History, + h_value: &mut History, history: &[TransactionState], - val: Option>, + value: Option>, extrinsic_index: u32, ) { let state = history.len() - 1; - if let Some((mut current, current_index)) = h.get_mut(history) { + if let Some((mut current, current_index)) = h_value.get_mut(history) { if current_index == state { - current.value = val; + current.value = value; current.extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); @@ -124,8 +122,8 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h.force_push(OverlayedValue { - value: val, + h_value.force_push(OverlayedValue { + value, extrinsics, }, state); } @@ -134,8 +132,8 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h.force_push(OverlayedValue { - value: val, + h_value.force_push(OverlayedValue { + value, extrinsics, }, state); @@ -146,21 +144,22 @@ impl OverlayedChangeSet { /// Garbage collect. fn gc(&mut self, eager: bool) { let eager = if eager { - let mut tx_ixs = Vec::new(); - for (i, h) in self.history.iter() { - if TransactionState::TxPending == h { - tx_ixs.push(i); + let mut transaction_index = Vec::new(); + for (i, state) in self.history.iter() { + if TransactionState::TxPending == state { + transaction_index.push(i); } } - Some(tx_ixs) + Some(transaction_index) } else { None }; let history = self.history.as_ref(); + let eager = || eager.as_ref().map(|t| t.as_slice()); // retain does change values - self.top.retain(|_, h| h.gc(history, eager.as_ref().map(|t| t.as_slice())).is_some()); + self.top.retain(|_, h_value| h_value.gc(history, eager()).is_some()); self.children.retain(|_, m| { - m.retain(|_, h| h.gc(history, eager.as_ref().map(|t| t.as_slice())).is_some()); + m.retain(|_, h_value| h_value.gc(history, eager()).is_some()); m.len() > 0 }); } @@ -324,9 +323,9 @@ impl OverlayedChanges { /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. pub fn storage(&self, key: &[u8]) -> Option> { - if let Some(overlay_val) = self.changes.top.get(key) { - if let Some(o_val) = overlay_val.get(self.changes.history.as_ref()) { - return Some(o_val.value.as_ref().map(|v| v.as_slice())) + if let Some(overlay_value) = self.changes.top.get(key) { + if let Some(o_value) = overlay_value.get(self.changes.history.as_ref()) { + return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } None @@ -337,9 +336,9 @@ impl OverlayedChanges { /// value has been set. pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.changes.children.get(storage_key) { - if let Some(overlay_val) = map.get(key) { - if let Some(o_val) = overlay_val.get(self.changes.history.as_ref()) { - return Some(o_val.value.as_ref().map(|v| v.as_slice())) + if let Some(overlay_value) = map.get(key) { + if let Some(o_value) = overlay_value.get(self.changes.history.as_ref()) { + return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } } @@ -349,22 +348,32 @@ impl OverlayedChanges { /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. - pub fn set_storage(&mut self, key: Vec, val: Option>) { - self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&val); + pub fn set_storage(&mut self, key: Vec, value: Option>) { + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&value); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), val, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), value, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_child_storage(&mut self, storage_key: Vec, key: Vec, val: Option>) { - self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&val); + pub(crate) fn set_child_storage( + &mut self, + storage_key: Vec, + key: Vec, + value: Option>, + ) { + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&value); let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); - set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), val, extrinsic_index); + set_with_extrinsic_overlayed_value( + entry, + self.changes.history.as_ref(), + value, + extrinsic_index, + ); } /// Clear child storage of given storage key. @@ -391,29 +400,29 @@ impl OverlayedChanges { pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); - let mut nb_remove = 0; + let mut number_removed = 0; for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - nb_remove += 1; + number_removed += 1; set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), None, extrinsic_index); } } - self.operation_from_last_gc += nb_remove; + self.operation_from_last_gc += number_removed; } pub(crate) fn clear_child_prefix(&mut self, storage_key: &[u8], prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); if let Some(child_change) = self.changes.children.get_mut(storage_key) { - let mut nb_remove = 0; + let mut number_removed = 0; for (key, entry) in child_change.iter_mut() { if key.starts_with(prefix) { - nb_remove += 1; + number_removed += 1; set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), None, extrinsic_index); } } - self.operation_from_last_gc += nb_remove; + self.operation_from_last_gc += number_removed; } } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 07ae1188c7b11..6d379f307f39c 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -41,6 +41,7 @@ type MemoryOnly = smallvec::SmallVec<[(V, usize); ALLOCATED_HISTORY]>; /// Size of preallocated history per element. /// Currently at two for committed and prospective only. /// It means that using transaction in a module got a direct allocation cost. +#[cfg(feature = "std")] const ALLOCATED_HISTORY: usize = 2; /// History of value that are related to a state history (eg field `history` of @@ -51,21 +52,20 @@ const ALLOCATED_HISTORY: usize = 2; #[cfg_attr(any(test, feature = "test"), derive(PartialEq))] pub struct History(MemoryOnly); - -impl History { - fn get_state(&self, index: usize) -> (&V, usize) { - (&self.0[index].0, self.0[index].1) - } -} - impl Default for History { fn default() -> Self { History(Default::default()) } } +// Following implementation are here to isolate +// buffer specific functions. impl History { + fn get_state(&self, index: usize) -> (&V, usize) { + (&self.0[index].0, self.0[index].1) + } + #[cfg(any(test, feature = "test"))] /// Create an history from an existing history. pub fn from_iter(input: impl IntoIterator) -> Self { @@ -103,21 +103,21 @@ impl History { /// This method shall only be call after a `get_mut` where /// the returned index indicate that a `set` will result /// in appending a value. - pub fn force_push(&mut self, val: V, tx_index: usize) { - self.0.push((val, tx_index)) + pub fn force_push(&mut self, value: V, history_index: usize) { + self.0.push((value, history_index)) } /// Set a value, it uses a state history as parameter. /// This method uses `get_mut` and do remove pending /// dropped value. - pub fn set(&mut self, history: &[TransactionState], val: V) { + pub fn set(&mut self, history: &[TransactionState], value: V) { if let Some((v, index)) = self.get_mut(history) { if index == history.len() - 1 { - *v = val; + *v = value; return; } } - self.force_push(val, history.len() - 1); + self.force_push(value, history.len() - 1); } fn mut_ref(&mut self, index: usize) -> &mut V { @@ -150,8 +150,8 @@ impl States { /// Build any state for testing only. #[cfg(any(test, feature = "test"))] - pub fn test_vector(test_state: Vec) -> Self { - States(test_state) + pub fn test_vector(test_states: Vec) -> Self { + States(test_states) } /// Discard prospective changes to state. @@ -368,20 +368,23 @@ impl History { /// Garbage collect a history, act as a `get_mut` with additional cost. - /// If `eager` is true, all dropped value are removed even if it means shifting - /// array byte. Otherwhise we mainly ensure collection up to last Commit state + /// To run `eager`, a `transaction_index` parameter of all `TxPending` states + /// must be provided, then all dropped value are removed even if it means shifting + /// array byte. Otherwhise we mainly garbage collect up to last Commit state /// (truncate left size). pub fn gc( &mut self, history: &[TransactionState], - tx_index: Option<&[usize]>, + transaction_index: Option<&[usize]>, ) -> Option<(&mut V, usize)> { - if let Some(tx_index) = tx_index { - let mut tx_index = &tx_index[..]; + if let Some(transaction_index) = transaction_index { + let mut transaction_index = &transaction_index[..]; let mut index = self.len(); if index == 0 { return None; } + // indicates that we got a value to return up to previous + // `TxPending` so values in between can be dropped. let mut bellow_value = usize::max_value(); let mut result: Option<(usize, usize)> = None; // internal method: should be use properly @@ -396,32 +399,34 @@ impl History { for _ in 0..index { self.remove(0); } - result = Some(result.map(|(i, hi)| (i - index, hi)).unwrap_or((0, history_index))); - index = 0; + result = Some(result.map(|(i, history_index)| (i - index, history_index)) + .unwrap_or((0, history_index))); + break; }, TransactionState::Pending | TransactionState::Prospective => { if history_index >= bellow_value { self.remove(index); - result.as_mut().map(|(i, _hi)| *i = *i - 1); + result.as_mut().map(|(i, _)| *i = *i - 1); } else { if result.is_none() { result = Some((index, history_index)); } + // move to next previous `TxPending` while bellow_value > history_index { - // bellow_value = pop - let split = tx_index.split_last() + // mut slice pop + let split = transaction_index.split_last() .map(|(v, sl)| (*v, sl)) .unwrap_or((0, &[])); bellow_value = split.0; - tx_index = split.1; + transaction_index = split.1; } } }, TransactionState::TxPending => { if history_index >= bellow_value { self.remove(index); - result.as_mut().map(|(i, _hi)| *i = *i - 1); + result.as_mut().map(|(i, _)| *i = *i - 1); } else { if result.is_none() { result = Some((index, history_index)); @@ -443,5 +448,3 @@ impl History { } } } - - From ea028ec2a96d3d640f672b8359f42d81698d68f9 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 28 Aug 2019 19:29:38 +0200 Subject: [PATCH 034/129] Change operation cost proto, add into_pending method and few formatting fixes. --- core/state-machine/src/overlayed_changes.rs | 16 ++--- core/utils/historied-data/src/lib.rs | 2 +- core/utils/historied-data/src/linear.rs | 78 +++++++++++++++------ 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index cad7235b4c4c2..e4b8f72681628 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -144,12 +144,7 @@ impl OverlayedChangeSet { /// Garbage collect. fn gc(&mut self, eager: bool) { let eager = if eager { - let mut transaction_index = Vec::new(); - for (i, state) in self.history.iter() { - if TransactionState::TxPending == state { - transaction_index.push(i); - } - } + let transaction_index = self.history.transaction_indexes(); Some(transaction_index) } else { None @@ -164,7 +159,6 @@ impl OverlayedChangeSet { }); } - /// Whether the change set is empty. pub fn is_empty(&self) -> bool { self.top.is_empty() && self.children.is_empty() @@ -349,7 +343,7 @@ impl OverlayedChanges { /// /// `None` can be used to delete a value specified by the given key. pub fn set_storage(&mut self, key: Vec, value: Option>) { - self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&value); + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), value, extrinsic_index); @@ -364,7 +358,7 @@ impl OverlayedChanges { key: Vec, value: Option>, ) { - self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(&value); + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); @@ -466,6 +460,10 @@ impl OverlayedChanges { /// Commit a transactional layer. pub fn commit_transaction(&mut self) { self.changes.commit_transaction(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { + self.operation_from_last_gc = 0; + self.gc(true); + } } /// Consume `OverlayedChanges` and take committed set. diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs index 33a0ef7022f32..5ccac6fb287ef 100644 --- a/core/utils/historied-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -70,7 +70,7 @@ pub struct GCConfiguration { impl GCConfiguration { /// Cost depending on value if any. - pub fn operation_cost(&self, val: &Option>) -> usize { + pub fn operation_cost(&self, val: Option<&rstd::vec::Vec>) -> usize { let additional_cost = if self.add_content_size_unit > 0 { if let Some(s) = val.as_ref() { s.len() / self.add_content_size_unit diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 6d379f307f39c..164ac9cc5b6e8 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -54,7 +54,7 @@ pub struct History(MemoryOnly); impl Default for History { fn default() -> Self { - History(Default::default()) + History(Default::default()) } } @@ -177,7 +177,7 @@ impl States { while i > 0 { i -= 1; match self.0[i] { - TransactionState::Dropped => (), + TransactionState::Dropped => (), TransactionState::Prospective | TransactionState::TxPending | TransactionState::Pending => self.0[i] = TransactionState::Committed, @@ -199,7 +199,7 @@ impl States { while i > 0 { i -= 1; match self.0[i] { - TransactionState::Dropped => (), + TransactionState::Dropped => (), TransactionState::Prospective | TransactionState::Pending => self.0[i] = TransactionState::Dropped, TransactionState::TxPending => { @@ -219,7 +219,7 @@ impl States { i -= 1; match self.0[i] { TransactionState::Prospective - | TransactionState::Dropped => (), + | TransactionState::Dropped => (), TransactionState::Pending => self.0[i] = TransactionState::Prospective, TransactionState::TxPending => { self.0[i] = TransactionState::Prospective; @@ -231,6 +231,19 @@ impl States { self.0.push(TransactionState::Pending); } + /// Return array of `TxPending` indexes in state. + /// This is use as an input for garbage collection. + pub fn transaction_indexes(&self) -> Vec { + let mut transaction_index = Vec::new(); + for (i, state) in self.0.iter().enumerate() { + if &TransactionState::TxPending == state { + transaction_index.push(i); + } + } + transaction_index + } + + } impl History { @@ -239,17 +252,17 @@ impl History { /// When possible please prefer `get_mut` as it can free /// some memory. pub fn get(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, + // index is never 0, let mut index = self.len(); if index == 0 { return None; } - debug_assert!(history.len() >= index); + debug_assert!(history.len() >= index); while index > 0 { index -= 1; let (v, history_index) = self.get_state(index); match history[history_index] { - TransactionState::Dropped => (), + TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective @@ -260,14 +273,39 @@ impl History { None } + /// Get latest value, consuming the historied data. + pub fn into_pending(mut self, history: &[TransactionState]) -> Option { + let mut index = self.len(); + if index == 0 { + return None; + } + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).1; + match history[history_index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending + | TransactionState::Prospective + | TransactionState::Committed => { + self.truncate(index + 1); + return self.pop().map(|v| v.0); + }, + } + } + None + } + + #[cfg(any(test, feature = "test"))] pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, + // index is never 0, let mut index = self.len(); if index == 0 { return None; } - debug_assert!(history.len() >= index); + debug_assert!(history.len() >= index); while index > 0 { index -= 1; let (v, history_index) = self.get_state(index); @@ -277,7 +315,7 @@ impl History { | TransactionState::Prospective => return Some(v), TransactionState::Committed - | TransactionState::Dropped => (), + | TransactionState::Dropped => (), } } None @@ -285,29 +323,28 @@ impl History { #[cfg(any(test, feature = "test"))] pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, + // index is never 0, let mut index = self.len(); if index == 0 { return None; } - debug_assert!(history.len() >= index); + debug_assert!(history.len() >= index); while index > 0 { index -= 1; let (v, history_index) = self.get_state(index); match history[history_index] { - TransactionState::Committed => - return Some(v), + TransactionState::Committed => return Some(v), TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective - | TransactionState::Dropped => (), + | TransactionState::Dropped => (), } } None } pub fn into_committed(mut self, history: &[TransactionState]) -> Option { - // index is never 0, + // index is never 0, let mut index = self.len(); if index == 0 { return None; @@ -315,7 +352,7 @@ impl History { // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= index); + debug_assert!(history.len() >= index); while index > 0 { index -= 1; let history_index = self.get_state(index).1; @@ -327,7 +364,7 @@ impl History { TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective - | TransactionState::Dropped => (), + | TransactionState::Dropped => (), } } None @@ -345,7 +382,7 @@ impl History { // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= index); + debug_assert!(history.len() >= index); while index > 0 { index -= 1; let history_index = self.get_state(index).1; @@ -390,7 +427,7 @@ impl History { // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= index); + debug_assert!(history.len() >= index); while index > 0 { index -= 1; let history_index = self.get_state(index).1; @@ -447,4 +484,5 @@ impl History { return self.get_mut(history); } } + } From 0ab789e720b4242fa5c43417ec084e13c490f190 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 29 Aug 2019 09:23:54 +0200 Subject: [PATCH 035/129] Doc and renaming 'force_push' to 'unsafe_push' (this function being pretty functionally unsafe). --- core/state-machine/src/overlayed_changes.rs | 6 +++--- core/test-runtime/Cargo.toml | 4 ++-- core/test-runtime/src/lib.rs | 2 ++ core/utils/historied-data/src/linear.rs | 6 +++--- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index e4b8f72681628..11d7aa114a2e3 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -79,7 +79,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, v)| (k, { let mut history = History::default(); - history.force_push(v, 0); + history.unsafe_push(v, 0); history })).collect(); result @@ -122,7 +122,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.force_push(OverlayedValue { + h_value.unsafe_push(OverlayedValue { value, extrinsics, }, state); @@ -132,7 +132,7 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.force_push(OverlayedValue { + h_value.unsafe_push(OverlayedValue { value, extrinsics, }, state); diff --git a/core/test-runtime/Cargo.toml b/core/test-runtime/Cargo.toml index efbea80564d24..a7056bb2f1181 100644 --- a/core/test-runtime/Cargo.toml +++ b/core/test-runtime/Cargo.toml @@ -18,7 +18,7 @@ aura-primitives = { package = "substrate-consensus-aura-primitives", path = "../ babe-primitives = { package = "substrate-consensus-babe-primitives", path = "../consensus/babe/primitives", default-features = false } rstd = { package = "sr-std", path = "../sr-std", default-features = false } runtime_io = { package = "sr-io", path = "../sr-io", default-features = false } -sr-primitives = { path = "../sr-primitives", default-features = false } +sr-primitives = { path = "../sr-primitives", default-features = false } session = { package = "substrate-session", path = "../session", default-features = false } runtime_version = { package = "sr-version", path = "../sr-version", default-features = false } runtime_support = { package = "srml-support", path = "../../srml/support", default-features = false } @@ -72,5 +72,5 @@ std = [ "srml-system/std", "app-crypto/std", "session/std", - "historied-data/std", + "historied-data/std", ] diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 8916fc533f634..08d67511ee67a 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -450,6 +450,7 @@ fn historied_data() -> u64 { let mut states = historied_data::linear::States::default(); let mut value = historied_data::linear::History::default(); if value.get(states.as_ref()) != None { + // value superior to 100 are error codes. return 101; } @@ -467,6 +468,7 @@ fn historied_data() -> u64 { return 104; } + // 0 for success return 0; } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 164ac9cc5b6e8..3e1ebd5a20087 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -71,7 +71,7 @@ impl History { pub fn from_iter(input: impl IntoIterator) -> Self { let mut history = History::default(); for v in input { - history.force_push(v.0, v.1); + history.unsafe_push(v.0, v.1); } history } @@ -103,7 +103,7 @@ impl History { /// This method shall only be call after a `get_mut` where /// the returned index indicate that a `set` will result /// in appending a value. - pub fn force_push(&mut self, value: V, history_index: usize) { + pub fn unsafe_push(&mut self, value: V, history_index: usize) { self.0.push((value, history_index)) } @@ -117,7 +117,7 @@ impl History { return; } } - self.force_push(value, history.len() - 1); + self.unsafe_push(value, history.len() - 1); } fn mut_ref(&mut self, index: usize) -> &mut V { From 117388ebb4ae6bc982541abe510e7c4625ea6977 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 5 Sep 2019 14:20:39 +0200 Subject: [PATCH 036/129] bump spec version --- node/runtime/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/runtime/src/lib.rs b/node/runtime/src/lib.rs index ded3fa483de8b..2fb74abf00cd2 100644 --- a/node/runtime/src/lib.rs +++ b/node/runtime/src/lib.rs @@ -80,8 +80,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to equal spec_version. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 155, - impl_version: 155, + spec_version: 156, + impl_version: 156, apis: RUNTIME_API_VERSIONS, }; From 838cd3b7d3c01f27e8eb14e7f2c8b1aa74c7324c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 12 Sep 2019 16:19:18 +0200 Subject: [PATCH 037/129] Addressing some nits, adding fuzz check into ci benches target. --- .gitlab-ci.yml | 4 ++ core/state-machine/benches/bench.rs | 6 +- core/state-machine/fuzz/src/lib.rs | 23 ++++---- core/state-machine/src/changes_trie/build.rs | 2 +- core/test-runtime/src/lib.rs | 59 ++++++++++---------- core/utils/historied-data/Cargo.toml | 2 - srml/support/src/storage/mod.rs | 1 - 7 files changed, 48 insertions(+), 49 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b67d5eb648720..7c4add7bb9bb7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -103,6 +103,10 @@ cargo-check-benches: script: - BUILD_DUMMY_WASM_BINARY=1 time cargo check --benches --all - sccache -s + - cd ./core/state-machine/fuzz + - time cargo check + - sccache -s + cargo-check-subkey: diff --git a/core/state-machine/benches/bench.rs b/core/state-machine/benches/bench.rs index 38f7c21520279..f8c8787e1e1f0 100644 --- a/core/state-machine/benches/bench.rs +++ b/core/state-machine/benches/bench.rs @@ -96,23 +96,21 @@ fn commit_drop_commit_and_get(b: &mut Bencher, input: &Vec) { }); } - - - fn bench_overlay_commit_drop_commit(c: &mut Criterion) { let inp = get_content(12, CONTENT_KEY_SIZE * 100); let inps = vec![inp]; c.bench_function_over_inputs("commit_drop_commit", commit_drop_commit, inps); } + fn bench_overlay_commit_drop_commit_get(c: &mut Criterion) { let inp = get_content(12, CONTENT_KEY_SIZE * 100); let inps = vec![inp]; c.bench_function_over_inputs("commit_drop_commit_get", commit_drop_commit_and_get, inps); } - criterion_group!(benches, bench_overlay_commit_drop_commit, bench_overlay_commit_drop_commit_get, ); + criterion_main!(benches); diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index 878cb7743064f..d4c4cf2d28472 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -239,17 +239,18 @@ impl RefOverlayedChanges { } } +// Those are error previously found through fuzzing. +// They were fixed, but we keep the vectors to check +// for regression without needing to run fuzzing. #[test] -fn debug_that() { -//50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255 - -//238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61 - -//50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67 - -//0x6e,0xff,0xf7,0x0,0x6e,0xff,0xff,0x2d,0xff,0xff,0xff,0xe - - let input = vec![ +fn previous_fuzzed_error() { + let inputs = [ + vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], + vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], + vec![50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67], + vec![0x6e, 0xff, 0xf7, 0x0, 0x6e, 0xff, 0xff, 0x2d, 0xff, 0xff, 0xff, 0xe], ]; - fuzz_transactions_inner(&input[..], true); + for input in inputs.iter() { + fuzz_transactions_inner(&input[..], true); + } } diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 6dbcd81f6131a..196dd4b3701e9 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -106,7 +106,7 @@ fn prepare_extrinsics_input<'a, B, H, Number>( block: block.clone(), storage_key: storage_key.to_vec(), }; - + let iter = prepare_extrinsics_input_inner(backend, block, changes, Some(storage_key.to_vec()))?; children_result.insert(child_index, iter); } diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 788421d1eb8b2..a57f7a9307f24 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -448,34 +448,6 @@ fn code_using_trie() -> u64 { } else { 103 } } -fn historied_data() -> u64 { - - let mut states = historied_data::linear::States::default(); - let mut value = historied_data::linear::History::default(); - if value.get(states.as_ref()) != None { - // value superior to 100 are error codes. - return 101; - } - - value.set(states.as_ref(), 42u64); - states.start_transaction(); - if value.get(states.as_ref()) != Some(&42) { - return 102; - } - value.set(states.as_ref(), 43u64); - if value.get(states.as_ref()) != Some(&43) { - return 103; - } - states.discard_transaction(); - if value.get(states.as_ref()) != Some(&42) { - return 104; - } - - // 0 for success - return 0; -} - - impl_opaque_keys! { pub struct SessionKeys { #[id(key_types::ED25519)] @@ -588,7 +560,7 @@ cfg_if! { } fn use_history_data() -> u64 { - historied_data() + test_historied_data() } fn use_transactions() -> u64 { @@ -784,7 +756,7 @@ cfg_if! { } fn use_history_data() -> u64 { - historied_data() + test_historied_data() } fn use_transactions() -> u64 { @@ -939,6 +911,33 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { (signature, public0) } +fn test_historied_data() -> u64 { + + let mut states = historied_data::linear::States::default(); + let mut value = historied_data::linear::History::default(); + if value.get(states.as_ref()) != None { + // value greater than 100 are error codes. + return 101; + } + + value.set(states.as_ref(), 42u64); + states.start_transaction(); + if value.get(states.as_ref()) != Some(&42) { + return 102; + } + value.set(states.as_ref(), 43u64); + if value.get(states.as_ref()) != Some(&43) { + return 103; + } + states.discard_transaction(); + if value.get(states.as_ref()) != Some(&42) { + return 104; + } + + // 0 for success + 0 +} + #[cfg(test)] mod tests { use substrate_test_runtime_client::{ diff --git a/core/utils/historied-data/Cargo.toml b/core/utils/historied-data/Cargo.toml index 79ab10dd8658b..6b5db5111c35d 100644 --- a/core/utils/historied-data/Cargo.toml +++ b/core/utils/historied-data/Cargo.toml @@ -9,8 +9,6 @@ edition = "2018" rstd = { package = "sr-std", path = "../../sr-std", default-features = false } smallvec = { version = "0.6", optional = true } -[dev-dependencies] - [features] default = ["std"] std = [ diff --git a/srml/support/src/storage/mod.rs b/srml/support/src/storage/mod.rs index 07b92f7b0d35a..5954e7f6260bc 100644 --- a/srml/support/src/storage/mod.rs +++ b/srml/support/src/storage/mod.rs @@ -30,7 +30,6 @@ pub mod generator; /// Execute under a transactional layer. /// -/// /// If the result of execution is an error, /// the transactional layer get reverted; otherwhise /// it is committed. From fe38b3002fa62beaa8c1980ccf5d4e9d0bfecb22 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 13 Sep 2019 09:52:55 +0200 Subject: [PATCH 038/129] Update iter docs, bump runtime version. --- core/state-machine/src/overlayed_changes.rs | 40 ++++++++++++--------- node/runtime/src/lib.rs | 4 +-- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 2c450f1653a55..8392e7a793c80 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -191,14 +191,16 @@ impl OverlayedChangeSet { self.history.commit_transaction(); } - /// Iterator over current state of the overlay. + /// Iterator over current state of the top level overlay, including change trie information. pub fn top_iter_overlay(&self) -> impl Iterator { self.top.iter() .filter_map(move |(k, v)| v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) } - /// Iterator over current state of the overlay. + /// Iterator over current state of a given child overlay, including change trie information. + /// Can be use to return top level storage by using a `None` storage key, + /// `top_iter_overlay` method should be use instead if possible. pub fn child_iter_overlay( &self, storage_key: Option<&[u8]>, @@ -216,12 +218,14 @@ impl OverlayedChangeSet { ) } - /// Iterator over current state of the overlay. + /// Iterator over current state of the top level overlay, values only. pub fn top_iter(&self) -> impl Iterator)> { self.top_iter_overlay().map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) } - /// Iterator over current state of the overlay. + /// Iterator over current state of a given child overlay, values only. + /// Can be use to return top level storage by using a `None` storage key, + /// `top_iter` method should be use instead if possible. pub fn child_iter( &self, storage_key: Option<&[u8]>, @@ -230,18 +234,7 @@ impl OverlayedChangeSet { .map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) } - /// Iterator over current state of the overlay. - pub fn children_iter_overlay( - &self, - ) -> impl Iterator)> { - self.children.iter() - .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() - .filter_map(move |(k, v)| - v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) - )) - } - - /// Iterator over current state of the overlay. + /// Iterator over current state of all children overlays, values only. pub fn children_iter( &self, ) -> impl Iterator)>)> { @@ -253,7 +246,8 @@ impl OverlayedChangeSet { )) } - /// Iterator over current state of the overlay. + /// Iterator over current state of all children overlays, values only. + /// Similar to `children_iter` but with key and value as `Vec`. pub fn owned_children_iter<'a>( &'a self, ) -> impl Iterator, impl Iterator, Option>)> + 'a)> + 'a { @@ -265,6 +259,17 @@ impl OverlayedChangeSet { )) } + /// Iterator over current state of all children overlays, including change trie information. + pub fn children_iter_overlay( + &self, + ) -> impl Iterator)> { + self.children.iter() + .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() + .filter_map(move |(k, v)| + v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) + )) + } + /// Test only method to access current prospective changes. /// It is here to keep old test compatibility and should be /// avoid for new tests. @@ -278,6 +283,7 @@ impl OverlayedChangeSet { } result } + /// Test only method to access current commited changes. /// It is here to keep old test compatibility and should be /// avoid for new tests. diff --git a/node/runtime/src/lib.rs b/node/runtime/src/lib.rs index 562dea598f5be..528bcc42a86f8 100644 --- a/node/runtime/src/lib.rs +++ b/node/runtime/src/lib.rs @@ -82,8 +82,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to equal spec_version. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 157, - impl_version: 157, + spec_version: 158, + impl_version: 158, apis: RUNTIME_API_VERSIONS, }; From 616fc2a6968842e16638f3195e180862a9f4ad11 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 13 Sep 2019 10:22:14 +0200 Subject: [PATCH 039/129] Simplify top and child iter into a single function. --- core/state-machine/fuzz/src/lib.rs | 2 +- core/state-machine/src/changes_trie/build.rs | 2 +- core/state-machine/src/ext.rs | 6 ++-- core/state-machine/src/lib.rs | 2 +- core/state-machine/src/overlayed_changes.rs | 31 ++++---------------- core/state-machine/src/testing.rs | 6 ++-- 6 files changed, 14 insertions(+), 35 deletions(-) diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index d4c4cf2d28472..c46df4c5f92cc 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -113,7 +113,7 @@ fn fuzz_transactions_inner(input: &[u8], check_gc: bool) { fn check_values(overlayed: &OverlayedChanges, ref_overlayed: &RefOverlayedChanges) -> (bool, usize) { let mut len = 0; let mut success = true; - for (key, value) in overlayed.top_iter() { + for (key, value) in overlayed.iter_values(None) { let ref_value = ref_overlayed.storage(key); if Some(value) != ref_value { diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 196dd4b3701e9..bd7749e12245a 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -128,7 +128,7 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( Number: BlockNumber, { - changes.changes.child_iter_overlay(storage_key.as_ref().map(|r| r.as_slice())) + changes.changes.iter_overlay(storage_key.as_ref().map(|r| r.as_slice())) .filter(|( _, v)| v.extrinsics.is_some()) .try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex, Vec)>, (k, v)| { match map.entry(k) { diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index fa9d659d4bee3..cf23c943d382a 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -164,7 +164,7 @@ where self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec())))) + .chain(self.overlay.changes.iter_values(None).map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec())))) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -306,7 +306,7 @@ where H: Hasher, let child_delta_iter = self.overlay.changes.owned_children_iter(); // compute and memoize - let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); + let delta = self.overlay.changes.iter_values(None).map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); let (root, transaction) = self.backend.full_storage_root(delta, child_delta_iter); self.storage_transaction = Some((transaction, root)); @@ -324,7 +324,7 @@ where H: Hasher, } else { let storage_key = storage_key.as_ref(); - let delta = self.overlay.changes.child_iter(Some(storage_key)) + let delta = self.overlay.changes.iter_values(Some(storage_key)) .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); let root = self.backend.child_storage_root(storage_key, delta).0; diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index e0cd4403403fc..cc23f32083997 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -903,7 +903,7 @@ mod tests { overlay.commit_prospective(); overlay.discard_prospective(); - let values: HashMap<_, _> = overlay.changes.top_iter() + let values: HashMap<_, _> = overlay.changes.iter_values(None) .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))).collect(); assert_eq!( values, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 8392e7a793c80..02ec642ef7ee9 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -191,17 +191,8 @@ impl OverlayedChangeSet { self.history.commit_transaction(); } - /// Iterator over current state of the top level overlay, including change trie information. - pub fn top_iter_overlay(&self) -> impl Iterator { - self.top.iter() - .filter_map(move |(k, v)| - v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) - } - - /// Iterator over current state of a given child overlay, including change trie information. - /// Can be use to return top level storage by using a `None` storage key, - /// `top_iter_overlay` method should be use instead if possible. - pub fn child_iter_overlay( + /// Iterator over current state of a given overlay, including change trie information. + pub fn iter_overlay( &self, storage_key: Option<&[u8]>, ) -> impl Iterator { @@ -218,19 +209,12 @@ impl OverlayedChangeSet { ) } - /// Iterator over current state of the top level overlay, values only. - pub fn top_iter(&self) -> impl Iterator)> { - self.top_iter_overlay().map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) - } - - /// Iterator over current state of a given child overlay, values only. - /// Can be use to return top level storage by using a `None` storage key, - /// `top_iter` method should be use instead if possible. - pub fn child_iter( + /// Iterator over current state of a given overlay, values only. + pub fn iter_values( &self, storage_key: Option<&[u8]>, ) -> impl Iterator)> { - self.child_iter_overlay(storage_key) + self.iter_overlay(storage_key) .map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) } @@ -545,11 +529,6 @@ impl OverlayedChanges { } } - /// Iterator over current state of the overlay. - pub fn top_iter(&self) -> impl Iterator)> { - self.changes.top_iter() - } - #[cfg(any(test, feature = "test"))] /// Count (slow) the number of key value, history included. /// Only for debugging or testing usage. diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index 029033694cb8f..e395e49bccc44 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -102,7 +102,7 @@ impl TestExternalities { /// Return a new backend with all pending value. pub fn commit_all(&self) -> InMemory { - let top = self.overlay.changes.top_iter_overlay() + let top = self.overlay.changes.iter_overlay(None) .map(|(k, v)| (None, k.to_vec(), v.value.clone())); let children = self.overlay.changes.children_iter_overlay() @@ -229,7 +229,7 @@ impl Externalities for TestExternalities where let child_delta_iter = self.overlay.changes.owned_children_iter(); // compute and memoize - let delta = self.overlay.changes.top_iter().map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); + let delta = self.overlay.changes.iter_values(None).map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); self.backend.full_storage_root(delta, child_delta_iter).0 } @@ -238,7 +238,7 @@ impl Externalities for TestExternalities where let storage_key = storage_key.as_ref(); let (root, is_empty, _) = { - let delta = self.overlay.changes.child_iter(Some(storage_key)) + let delta = self.overlay.changes.iter_values(Some(storage_key)) .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); self.backend.child_storage_root(storage_key, delta) From 7001e8ca9f61a4eca9f60964fd6c674f2c300fcf Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 13 Sep 2019 10:58:30 +0200 Subject: [PATCH 040/129] Restore access to iter for fuzzer. --- core/state-machine/src/overlayed_changes.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 02ec642ef7ee9..8b1a219739a14 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -529,6 +529,15 @@ impl OverlayedChanges { } } + #[cfg(any(test, feature = "test"))] + /// Iterator over current state of the overlay. + pub fn iter_values( + &self, + storage_key: Option<&[u8]>, + ) -> impl Iterator)> { + self.changes.iter_values(storage_key) + } + #[cfg(any(test, feature = "test"))] /// Count (slow) the number of key value, history included. /// Only for debugging or testing usage. From a84b8db426d858a186c86cca37ef1379cf24ca70 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 16 Sep 2019 21:07:23 +0200 Subject: [PATCH 041/129] rename gc for liner to get_mut_pruning, rename unsafe_push to push_unchecked --- .gitlab-ci.yml | 1 - core/state-machine/src/overlayed_changes.rs | 10 +++---- core/utils/historied-data/src/linear.rs | 31 ++++++++++----------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ddb41a85e3a61..40d3a684516a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -102,7 +102,6 @@ cargo-check-benches: <<: *docker-env script: - BUILD_DUMMY_WASM_BINARY=1 time cargo +nightly check --benches --all - - sccache -s - cd ./core/state-machine/fuzz - time cargo check - sccache -s diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 8b1a219739a14..a0a79a9e05532 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -80,7 +80,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, v)| (k, { let mut history = History::default(); - history.unsafe_push(v, 0); + history.push_unchecked(v, 0); history })).collect(); result @@ -123,7 +123,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.unsafe_push(OverlayedValue { + h_value.push_unchecked(OverlayedValue { value, extrinsics, }, state); @@ -133,7 +133,7 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.unsafe_push(OverlayedValue { + h_value.push_unchecked(OverlayedValue { value, extrinsics, }, state); @@ -153,9 +153,9 @@ impl OverlayedChangeSet { let history = self.history.as_ref(); let eager = || eager.as_ref().map(|t| t.as_slice()); // retain does change values - self.top.retain(|_, h_value| h_value.gc(history, eager()).is_some()); + self.top.retain(|_, h_value| h_value.get_mut_pruning(history, eager()).is_some()); self.children.retain(|_, m| { - m.retain(|_, h_value| h_value.gc(history, eager()).is_some()); + m.retain(|_, h_value| h_value.get_mut_pruning(history, eager()).is_some()); m.len() > 0 }); } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 3e1ebd5a20087..12420d0ca840c 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -8,11 +8,11 @@ // Substrate is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . +// along with Substrate. If not, see . //! Transactional overlay implementation. //! @@ -71,7 +71,7 @@ impl History { pub fn from_iter(input: impl IntoIterator) -> Self { let mut history = History::default(); for v in input { - history.unsafe_push(v.0, v.1); + history.push_unchecked(v.0, v.1); } history } @@ -103,7 +103,7 @@ impl History { /// This method shall only be call after a `get_mut` where /// the returned index indicate that a `set` will result /// in appending a value. - pub fn unsafe_push(&mut self, value: V, history_index: usize) { + pub fn push_unchecked(&mut self, value: V, history_index: usize) { self.0.push((value, history_index)) } @@ -117,7 +117,7 @@ impl History { return; } } - self.unsafe_push(value, history.len() - 1); + self.push_unchecked(value, history.len() - 1); } fn mut_ref(&mut self, index: usize) -> &mut V { @@ -372,7 +372,7 @@ impl History { /// Access to latest pending value (non dropped state in history). /// - /// This method remove latest dropped value up to the latest valid value. + /// This method removes latest dropped values up to the latest valid value. pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { let mut index = self.len(); @@ -404,25 +404,24 @@ impl History { } - /// Garbage collect a history, act as a `get_mut` with additional cost. + /// Garbage collect the history, act as a `get_mut` with additional cost. /// To run `eager`, a `transaction_index` parameter of all `TxPending` states /// must be provided, then all dropped value are removed even if it means shifting /// array byte. Otherwhise we mainly garbage collect up to last Commit state /// (truncate left size). - pub fn gc( + pub fn get_mut_pruning( &mut self, history: &[TransactionState], transaction_index: Option<&[usize]>, ) -> Option<(&mut V, usize)> { - if let Some(transaction_index) = transaction_index { - let mut transaction_index = &transaction_index[..]; + if let Some(mut transaction_index) = transaction_index { let mut index = self.len(); if index == 0 { return None; } // indicates that we got a value to return up to previous // `TxPending` so values in between can be dropped. - let mut bellow_value = usize::max_value(); + let mut below_value = usize::max_value(); let mut result: Option<(usize, usize)> = None; // internal method: should be use properly // (history of the right overlay change set @@ -442,7 +441,7 @@ impl History { }, TransactionState::Pending | TransactionState::Prospective => { - if history_index >= bellow_value { + if history_index >= below_value { self.remove(index); result.as_mut().map(|(i, _)| *i = *i - 1); } else { @@ -450,18 +449,18 @@ impl History { result = Some((index, history_index)); } // move to next previous `TxPending` - while bellow_value > history_index { + while below_value > history_index { // mut slice pop let split = transaction_index.split_last() .map(|(v, sl)| (*v, sl)) .unwrap_or((0, &[])); - bellow_value = split.0; + below_value = split.0; transaction_index = split.1; } } }, TransactionState::TxPending => { - if history_index >= bellow_value { + if history_index >= below_value { self.remove(index); result.as_mut().map(|(i, _)| *i = *i - 1); } else { @@ -469,7 +468,7 @@ impl History { result = Some((index, history_index)); } } - bellow_value = usize::max_value(); + below_value = usize::max_value(); }, TransactionState::Dropped => { self.remove(index); From 9787a97804e514e7ae50cb649c22b3a58e11a1c5 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 16 Sep 2019 21:51:24 +0200 Subject: [PATCH 042/129] switch tuple to historiedvalue struct. --- core/state-machine/src/changes_trie/build.rs | 16 ++--- core/state-machine/src/ext.rs | 6 +- core/state-machine/src/overlayed_changes.rs | 28 +++++---- core/utils/historied-data/src/linear.rs | 65 ++++++++++++-------- 4 files changed, 67 insertions(+), 48 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index bd7749e12245a..6b924477ef702 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -325,7 +325,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historied_data::linear::{History, States}; + use historied_data::linear::{History, States, HistoriedValue}; use historied_data::State as TransactionState; use super::*; @@ -404,7 +404,7 @@ mod test { value: Some(3u32.encode()), extrinsics: None, }, 0), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), (vec![100], History::from_iter(vec![ (OverlayedValue { value: Some(vec![202]), @@ -414,19 +414,19 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, 1), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), (vec![101], History::from_iter(vec![ (OverlayedValue { value: Some(vec![203]), extrinsics: Some(vec![1].into_iter().collect()) }, 0), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), (vec![103], History::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![0, 1].into_iter().collect()) }, 1), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), ].into_iter().collect(), children: vec![ (child_trie_key1, vec![ @@ -439,7 +439,7 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, 1), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), ].into_iter().collect()), (child_trie_key2, vec![ (vec![100], History::from_iter(vec![ @@ -447,7 +447,7 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) }, 1), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), ].into_iter().collect()), ].into_iter().collect(), }, @@ -649,7 +649,7 @@ mod test { value: None, extrinsics: Some(vec![1].into_iter().collect()), }, 1), - ])); + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))); let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; let changes_trie_nodes = prepare_input( diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index cf23c943d382a..816e3d4a5b500 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -382,7 +382,7 @@ mod tests { use crate::changes_trie::{Configuration as ChangesTrieConfiguration, InMemoryStorage as InMemoryChangesTrieStorage}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historied_data::linear::History; + use historied_data::linear::{History, HistoriedValue}; use super::*; type TestBackend = InMemory; @@ -405,13 +405,13 @@ mod tests { value: Some(3u32.encode()), extrinsics: Some(vec![1].into_iter().collect()) }, 0), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), (vec![1], History::from_iter(vec![ (OverlayedValue { value: Some(vec![100]), extrinsics: Some(vec![1].into_iter().collect()) }, 0), - ])), + ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), ].into_iter().collect(), }, operation_from_last_gc: 0, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index a0a79a9e05532..b3d1e89a33556 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,7 +22,7 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use historied_data::linear::{States, History}; +use historied_data::linear::{States, History, HistoriedValue}; use historied_data::State as TransactionState; use historied_data::DEFAULT_GC_CONF; @@ -78,9 +78,9 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { let mut result = OverlayedChangeSet::default(); - result.top = iter.into_iter().map(|(k, v)| (k, { + result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = History::default(); - history.push_unchecked(v, 0); + history.push_unchecked(HistoriedValue { value, index: 0 }); history })).collect(); result @@ -123,20 +123,26 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.push_unchecked(OverlayedValue { - value, - extrinsics, - }, state); + h_value.push_unchecked(HistoriedValue { + value: OverlayedValue { + value, + extrinsics, + }, + index: state, + }); } } else { let mut extrinsics: Option> = None; extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.push_unchecked(OverlayedValue { - value, - extrinsics, - }, state); + h_value.push_unchecked(HistoriedValue { + value: OverlayedValue { + value, + extrinsics, + }, + index: state, + }); } } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 12420d0ca840c..06caa9ebaea91 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -26,17 +26,27 @@ use crate::State as TransactionState; use rstd::vec::Vec; use rstd::vec; +/// An entry at a given history height. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct HistoriedValue { + /// The stored value. + pub value: V, + /// The moment in history when the value got set. + pub index: usize, +} + /// Array like buffer for in memory storage. /// By in memory we expect that this will /// not required persistence and is not serialized. #[cfg(not(feature = "std"))] -type MemoryOnly = Vec<(V, usize)>; +type MemoryOnly = Vec>; /// Array like buffer for in memory storage. /// By in memory we expect that this will /// not required persistence and is not serialized. #[cfg(feature = "std")] -type MemoryOnly = smallvec::SmallVec<[(V, usize); ALLOCATED_HISTORY]>; +type MemoryOnly = smallvec::SmallVec<[HistoriedValue; ALLOCATED_HISTORY]>; /// Size of preallocated history per element. /// Currently at two for committed and prospective only. @@ -62,16 +72,16 @@ impl Default for History { // buffer specific functions. impl History { - fn get_state(&self, index: usize) -> (&V, usize) { - (&self.0[index].0, self.0[index].1) + fn get_state(&self, index: usize) -> &HistoriedValue { + &self.0[index] } #[cfg(any(test, feature = "test"))] /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator) -> Self { + pub fn from_iter(input: impl IntoIterator>) -> Self { let mut history = History::default(); for v in input { - history.push_unchecked(v.0, v.1); + history.push_unchecked(v); } history } @@ -90,7 +100,7 @@ impl History { self.0.truncate(index) } - fn pop(&mut self) -> Option<(V, usize)> { + fn pop(&mut self) -> Option> { self.0.pop() } @@ -103,8 +113,8 @@ impl History { /// This method shall only be call after a `get_mut` where /// the returned index indicate that a `set` will result /// in appending a value. - pub fn push_unchecked(&mut self, value: V, history_index: usize) { - self.0.push((value, history_index)) + pub fn push_unchecked(&mut self, value: HistoriedValue) { + self.0.push(value) } /// Set a value, it uses a state history as parameter. @@ -117,11 +127,14 @@ impl History { return; } } - self.push_unchecked(value, history.len() - 1); + self.push_unchecked(HistoriedValue{ + value, + index: history.len() - 1, + }); } fn mut_ref(&mut self, index: usize) -> &mut V { - &mut self.0[index].0 + &mut self.0[index].value } } @@ -260,14 +273,14 @@ impl History { debug_assert!(history.len() >= index); while index > 0 { index -= 1; - let (v, history_index) = self.get_state(index); - match history[history_index] { + let HistoriedValue { value, index: history_index } = self.get_state(index); + match history[*history_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective | TransactionState::Committed => - return Some(v), + return Some(value), } } None @@ -282,7 +295,7 @@ impl History { debug_assert!(history.len() >= index); while index > 0 { index -= 1; - let history_index = self.get_state(index).1; + let history_index = self.get_state(index).index; match history[history_index] { TransactionState::Dropped => (), TransactionState::Pending @@ -290,7 +303,7 @@ impl History { | TransactionState::Prospective | TransactionState::Committed => { self.truncate(index + 1); - return self.pop().map(|v| v.0); + return self.pop().map(|v| v.value); }, } } @@ -308,12 +321,12 @@ impl History { debug_assert!(history.len() >= index); while index > 0 { index -= 1; - let (v, history_index) = self.get_state(index); - match history[history_index] { + let HistoriedValue { value, index: history_index } = self.get_state(index); + match history[*history_index] { TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective => - return Some(v), + return Some(value), TransactionState::Committed | TransactionState::Dropped => (), } @@ -331,9 +344,9 @@ impl History { debug_assert!(history.len() >= index); while index > 0 { index -= 1; - let (v, history_index) = self.get_state(index); - match history[history_index] { - TransactionState::Committed => return Some(v), + let HistoriedValue { value, index: history_index } = self.get_state(index); + match history[*history_index] { + TransactionState::Committed => return Some(value), TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective @@ -355,11 +368,11 @@ impl History { debug_assert!(history.len() >= index); while index > 0 { index -= 1; - let history_index = self.get_state(index).1; + let history_index = self.get_state(index).index; match history[history_index] { TransactionState::Committed => { self.truncate(index + 1); - return self.pop().map(|v| v.0); + return self.pop().map(|v| v.value); }, TransactionState::Pending | TransactionState::TxPending @@ -385,7 +398,7 @@ impl History { debug_assert!(history.len() >= index); while index > 0 { index -= 1; - let history_index = self.get_state(index).1; + let history_index = self.get_state(index).index; match history[history_index] { TransactionState::Committed => { // here we could gc all preceding values but that is additional cost @@ -429,7 +442,7 @@ impl History { debug_assert!(history.len() >= index); while index > 0 { index -= 1; - let history_index = self.get_state(index).1; + let history_index = self.get_state(index).index; match history[history_index] { TransactionState::Committed => { for _ in 0..index { From d9d21185c012f14687d7a322fda1d9bced8af570 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 16 Sep 2019 22:16:36 +0200 Subject: [PATCH 043/129] use len instead of internal_item_counts --- core/state-machine/src/overlayed_changes.rs | 2 +- core/utils/historied-data/src/linear.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index b3d1e89a33556..4f20e01ae4a69 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -550,7 +550,7 @@ impl OverlayedChanges { pub fn top_count_keyvalue_pair(&self) -> usize { let mut result = 0; for (_, v) in self.changes.top.iter() { - result += v.internal_item_counts() + result += v.len() } result } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 06caa9ebaea91..d9b85d411262e 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -87,15 +87,16 @@ impl History { } #[cfg(any(test, feature = "test"))] - /// Debugging function for test and fuzzing. - pub fn internal_item_counts(&self) -> usize { + pub fn len(&self) -> usize { self.0.len() } + #[cfg(not(any(test, feature = "test")))] fn len(&self) -> usize { self.0.len() } + fn truncate(&mut self, index: usize) { self.0.truncate(index) } From 887f95b789e17a6d1090f56160687726512af382 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Tue, 17 Sep 2019 12:15:19 +0200 Subject: [PATCH 044/129] historied value on get and get mut to. --- core/state-machine/src/overlayed_changes.rs | 10 ++--- core/utils/historied-data/src/linear.rs | 46 ++++++++++++++------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 4f20e01ae4a69..e6f732fba29c7 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -112,15 +112,15 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsic_index: u32, ) { let state = history.len() - 1; - if let Some((mut current, current_index)) = h_value.get_mut(history) { + if let Some(current) = h_value.get_mut(history) { - if current_index == state { - current.value = value; - current.extrinsics.get_or_insert_with(Default::default) + if current.index == state { + current.value.value = value; + current.value.extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); } else { - let mut extrinsics = current.extrinsics.clone(); + let mut extrinsics = current.value.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); h_value.push_unchecked(HistoriedValue { diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index d9b85d411262e..42465baeb578e 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -36,6 +36,21 @@ pub struct HistoriedValue { pub index: usize, } +impl From<(V, usize)> for HistoriedValue { + fn from(input: (V, usize)) -> HistoriedValue { + HistoriedValue { value: input.0, index: input.1 } + } +} + +impl HistoriedValue { + fn as_ref(&self) -> HistoriedValue<&V> { + HistoriedValue { + value: &self.value, + index: self.index, + } + } +} + /// Array like buffer for in memory storage. /// By in memory we expect that this will /// not required persistence and is not serialized. @@ -72,8 +87,8 @@ impl Default for History { // buffer specific functions. impl History { - fn get_state(&self, index: usize) -> &HistoriedValue { - &self.0[index] + fn get_state(&self, index: usize) -> HistoriedValue<&V> { + self.0[index].as_ref() } #[cfg(any(test, feature = "test"))] @@ -96,7 +111,6 @@ impl History { self.0.len() } - fn truncate(&mut self, index: usize) { self.0.truncate(index) } @@ -122,13 +136,13 @@ impl History { /// This method uses `get_mut` and do remove pending /// dropped value. pub fn set(&mut self, history: &[TransactionState], value: V) { - if let Some((v, index)) = self.get_mut(history) { - if index == history.len() - 1 { - *v = value; + if let Some(v) = self.get_mut(history) { + if v.index == history.len() - 1 { + *v.value = value; return; } } - self.push_unchecked(HistoriedValue{ + self.push_unchecked(HistoriedValue { value, index: history.len() - 1, }); @@ -275,7 +289,7 @@ impl History { while index > 0 { index -= 1; let HistoriedValue { value, index: history_index } = self.get_state(index); - match history[*history_index] { + match history[history_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending @@ -323,7 +337,7 @@ impl History { while index > 0 { index -= 1; let HistoriedValue { value, index: history_index } = self.get_state(index); - match history[*history_index] { + match history[history_index] { TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective => @@ -346,7 +360,7 @@ impl History { while index > 0 { index -= 1; let HistoriedValue { value, index: history_index } = self.get_state(index); - match history[*history_index] { + match history[history_index] { TransactionState::Committed => return Some(value), TransactionState::Pending | TransactionState::TxPending @@ -387,7 +401,7 @@ impl History { /// Access to latest pending value (non dropped state in history). /// /// This method removes latest dropped values up to the latest valid value. - pub fn get_mut(&mut self, history: &[TransactionState]) -> Option<(&mut V, usize)> { + pub fn get_mut(&mut self, history: &[TransactionState]) -> Option> { let mut index = self.len(); if index == 0 { @@ -404,12 +418,12 @@ impl History { TransactionState::Committed => { // here we could gc all preceding values but that is additional cost // and get_mut should stop at pending following committed. - return Some((self.mut_ref(index), history_index)) + return Some((self.mut_ref(index), history_index).into()) }, TransactionState::Pending | TransactionState::TxPending | TransactionState::Prospective => { - return Some((self.mut_ref(index), history_index)) + return Some((self.mut_ref(index), history_index).into()) }, TransactionState::Dropped => { let _ = self.pop(); }, } @@ -427,7 +441,7 @@ impl History { &mut self, history: &[TransactionState], transaction_index: Option<&[usize]>, - ) -> Option<(&mut V, usize)> { + ) -> Option> { if let Some(mut transaction_index) = transaction_index { let mut index = self.len(); if index == 0 { @@ -490,11 +504,11 @@ impl History { } } if let Some((index, history_index)) = result { - Some((self.mut_ref(index), history_index)) + Some((self.mut_ref(index), history_index).into()) } else { None } } else { - return self.get_mut(history); + self.get_mut(history) } } From ff4bcb096e24892c54b23de48308d8c6446241ca Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 18 Sep 2019 11:45:40 +0200 Subject: [PATCH 045/129] proper rand on bench --- Cargo.lock | 12 +++++++++++- core/state-machine/Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2bae170b2d47a..aaef34664a21c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3204,6 +3204,7 @@ dependencies = [ "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3301,6 +3302,15 @@ dependencies = [ "rand_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand_pcg" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand_xorshift" version = "0.1.1" @@ -5343,7 +5353,6 @@ dependencies = [ "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-panic-handler 2.0.0", "substrate-primitives 2.0.0", @@ -6794,6 +6803,7 @@ dependencies = [ "checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" "checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" "checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +"checksum rand_pcg 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e196346cbbc5c70c77e7b4926147ee8e383a38ee4d15d58a08098b169e492b6" "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" "checksum rand_xoshiro 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "03b418169fb9c46533f326efd6eed2576699c44ca92d3052a066214a8d828929" "checksum rayon 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a4b0186e22767d5b9738a05eab7c6ac90b15db17e5b5f9bd87976dd7d89a10a4" diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index 2dadeee3895ff..e41b0f325e943 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -22,7 +22,7 @@ rand = "0.7.1" [dev-dependencies] hex-literal = "0.2.0" criterion = "0.2" -rand = "0.6" +rand = { version = "0.7.1", features = ["small_rng"] } historied-data = { path = "../utils/historied-data", features = ["std", "test"] } [[bench]] From 45dc9357b65715286c408075a568be3ad30a859f Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 20 Sep 2019 11:06:52 +0200 Subject: [PATCH 046/129] Update core/utils/historied-data/README.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Bastian Köcher --- core/utils/historied-data/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/historied-data/README.md b/core/utils/historied-data/README.md index 8d469fbb00b56..b8ef518428c5d 100644 --- a/core/utils/historied-data/README.md +++ b/core/utils/historied-data/README.md @@ -11,7 +11,7 @@ history context. History is serialize as a per item basis. -This crates is be `no_std` compatible, unless feature `std` is used. +This crates is `no_std` compatible as long as the `std` feature is not enabled. For more information see From d7c9f8a1e3b692a6df0f48cff6a0c032b521e68a Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 20 Sep 2019 11:07:24 +0200 Subject: [PATCH 047/129] Update core/utils/historied-data/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Bastian Köcher --- core/utils/historied-data/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs index 5ccac6fb287ef..54da427320a17 100644 --- a/core/utils/historied-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// Copyright 2019 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify From 6c9abda8850529de77217bd5409b9a122ceb61a8 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 20 Sep 2019 11:07:37 +0200 Subject: [PATCH 048/129] Update core/utils/historied-data/src/linear.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Bastian Köcher --- core/utils/historied-data/src/linear.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 261300813e255..4ebb43a7f8b99 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -1,4 +1,4 @@ -// Copyright 2017-2019 Parity Technologies (UK) Ltd. +// Copyright 2019 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify From 89b4f26e537ac85be531746f6b19bd4dce125ba5 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 20 Sep 2019 12:11:20 +0200 Subject: [PATCH 049/129] Switching runtime test to panic instead of returning result. Some documentation improvment from review. --- core/executor/src/wasm_executor.rs | 9 +++----- core/primitives/src/traits.rs | 2 +- core/sr-api-macros/tests/runtime_calls.rs | 2 +- core/state-machine/fuzz/src/lib.rs | 4 +--- core/state-machine/src/overlayed_changes.rs | 21 ++++++++++--------- core/test-runtime/src/lib.rs | 23 ++++++++------------- 6 files changed, 26 insertions(+), 35 deletions(-) diff --git a/core/executor/src/wasm_executor.rs b/core/executor/src/wasm_executor.rs index 417b103d68736..a57063454f5bd 100644 --- a/core/executor/src/wasm_executor.rs +++ b/core/executor/src/wasm_executor.rs @@ -1362,22 +1362,19 @@ impl_wasm_host_interface! { ext_storage_start_transaction() { with_external_storage(move || Ok(runtime_io::storage_start_transaction()) - )?; - Ok(()) + ) } ext_storage_discard_transaction() { with_external_storage(move || Ok(runtime_io::storage_discard_transaction()) - )?; - Ok(()) + ) } ext_storage_commit_transaction() { with_external_storage(move || Ok(runtime_io::storage_commit_transaction()) - )?; - Ok(()) + ) } } } diff --git a/core/primitives/src/traits.rs b/core/primitives/src/traits.rs index 445e23a00b476..83759abb4fa11 100644 --- a/core/primitives/src/traits.rs +++ b/core/primitives/src/traits.rs @@ -181,7 +181,7 @@ pub trait Externalities { /// Create a new transactional layer. fn storage_start_transaction(&mut self); - /// Discard a transactional layer, pending changes of every trasaction below this layer are + /// Discard a transactional layer, pending changes of every transaction behind this layer are /// dropped (including committed changes) . fn storage_discard_transaction(&mut self); diff --git a/core/sr-api-macros/tests/runtime_calls.rs b/core/sr-api-macros/tests/runtime_calls.rs index 6f5dedbf886dd..017659c20ec3d 100644 --- a/core/sr-api-macros/tests/runtime_calls.rs +++ b/core/sr-api-macros/tests/runtime_calls.rs @@ -132,7 +132,7 @@ fn use_history_data() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); let runtime_api = client.runtime_api(); let block_id = BlockId::Number(client.info().chain.best_number); - assert_eq!(runtime_api.use_history_data(&block_id).unwrap(), 0); + assert!(runtime_api.use_history_data(&block_id).is_ok()); } #[test] diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index c46df4c5f92cc..9090ef447ef08 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -15,9 +15,7 @@ // along with Substrate. If not, see . //! Substrate state machine fuzzing implementations. -use substrate_state_machine::{ - OverlayedChanges, -}; +use substrate_state_machine::OverlayedChanges; use std::collections::HashMap; /// Size of key, max 255 diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index e6f732fba29c7..361adcb4a77f1 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -38,8 +38,9 @@ pub struct OverlayedChanges { /// runtime if it supports change tries. pub(crate) changes_trie_config: Option, /// Counter of number of operation between garbage collection. - /// Add or delete cost one, additional cost per size by counting a fix size - /// as a unit. + /// Changing or deleting a value increment this counter by one. + /// An additional cost for data added is added for every n bytes of data. + /// n is currently defined by `DEFAULT_GC_CONF`. pub(crate) operation_from_last_gc: usize, } @@ -56,12 +57,14 @@ pub struct OverlayedValue { /// Overlayed change set, keep history of values. /// -/// This does not work by stacking hashmap for transaction, -/// but store history of value instead. +/// This does not work by stacking hashmaps per transaction, +/// but by storing history of a value instead. /// Value validity is given by a global indexed state history. -/// When dropping or committing a layer of transaction, -/// history for each values is kept until -/// next mutable access to the value. +/// When dropping or committing a transaction layer, +/// full history for each values is kept until +/// next garbage collection. +/// Note that on mutable access terminal values corresponding +/// to a deleted layer are also dropped. #[derive(Debug, Clone, Default)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { @@ -113,12 +116,10 @@ fn set_with_extrinsic_inner_overlayed_value( ) { let state = history.len() - 1; if let Some(current) = h_value.get_mut(history) { - if current.index == state { current.value.value = value; current.value.extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - } else { let mut extrinsics = current.value.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) @@ -162,7 +163,7 @@ impl OverlayedChangeSet { self.top.retain(|_, h_value| h_value.get_mut_pruning(history, eager()).is_some()); self.children.retain(|_, m| { m.retain(|_, h_value| h_value.get_mut_pruning(history, eager()).is_some()); - m.len() > 0 + !m.is_empty() }); } diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 6046aa570d86e..3aec063efc3dc 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -261,7 +261,7 @@ cfg_if! { /// Trie no_std testing. fn use_trie() -> u64; /// History data no_std testing. - fn use_history_data() -> u64; + fn use_history_data(); /// Transactional tests. fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; @@ -309,7 +309,7 @@ cfg_if! { /// trie no_std testing fn use_trie() -> u64; /// History data no_std testing. - fn use_history_data() -> u64; + fn use_history_data(); /// Transactional tests. fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; @@ -563,7 +563,7 @@ cfg_if! { code_using_trie() } - fn use_history_data() -> u64 { + fn use_history_data() { test_historied_data() } @@ -764,7 +764,7 @@ cfg_if! { code_using_trie() } - fn use_history_data() -> u64 { + fn use_history_data() { test_historied_data() } @@ -925,31 +925,26 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { (signature, public0) } -fn test_historied_data() -> u64 { - +fn test_historied_data() { let mut states = historied_data::linear::States::default(); let mut value = historied_data::linear::History::default(); if value.get(states.as_ref()) != None { - // value greater than 100 are error codes. - return 101; + panic!("Got a value for empty data"); } value.set(states.as_ref(), 42u64); states.start_transaction(); if value.get(states.as_ref()) != Some(&42) { - return 102; + panic!("Got a wrong result accessing a one element data"); } value.set(states.as_ref(), 43u64); if value.get(states.as_ref()) != Some(&43) { - return 103; + panic!("Got a wrong result accessing a two element data"); } states.discard_transaction(); if value.get(states.as_ref()) != Some(&42) { - return 104; + panic!("Got a wrong result accessing a one element data after a discard"); } - - // 0 for success - 0 } fn test_read_storage() { From e076d128cfaa413d963a758c094f4f333f7689ec Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 10:16:09 +0200 Subject: [PATCH 050/129] Avoid clone explicitely in some case (the clone with this PR is a lot more costy than previously). Could also skip this call by using a double prospective on the overlay. --- core/state-machine/src/lib.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 47bfd727878b2..51b800208331c 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -413,14 +413,13 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where init_overlay(self.overlay, false, &self.backend)?; let result = { - let orig_changes = self.overlay.changes.clone(); let (result, storage_delta, changes_delta) = match manager { ExecutionManager::Both(on_consensus_failure) => { self.execute_call_with_both_strategy( compute_tx, native_call.take(), - orig_changes, + self.overlay.changes.clone(), on_consensus_failure, ) }, @@ -428,7 +427,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where self.execute_call_with_native_else_wasm_strategy( compute_tx, native_call.take(), - orig_changes, + self.overlay.changes.clone(), ) }, ExecutionManager::AlwaysWasm(trust_level) => { From b877213283d11aebce4025c8b3e8026a888c23d5 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 10 Oct 2019 10:28:42 +0200 Subject: [PATCH 051/129] remove it from parameter. --- core/state-machine/src/lib.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 51b800208331c..b30f04fd78f17 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -26,7 +26,6 @@ use primitives::{ storage::well_known_keys, NativeOrEncoded, NeverNativeValue, offchain::OffchainExt, traits::{KeystoreExt, CodeExecutor}, hexdisplay::HexDisplay, hash::H256, }; -use overlayed_changes::OverlayedChangeSet; use externalities::Extensions; pub mod backend; @@ -299,7 +298,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where &mut self, compute_tx: bool, mut native_call: Option, - orig_changes: OverlayedChangeSet, on_consensus_failure: Handler, ) -> ( CallResult, @@ -313,6 +311,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where CallResult, ) -> CallResult { + let orig_changes = self.overlay.changes.clone(); let (result, was_native, storage_delta, changes_delta) = self.execute_aux( compute_tx, true, @@ -320,7 +319,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where ); if was_native { - self.overlay.changes = orig_changes.clone(); + self.overlay.changes = orig_changes; let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, false, @@ -344,7 +343,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where &mut self, compute_tx: bool, mut native_call: Option, - orig_changes: OverlayedChangeSet, ) -> ( CallResult, Option<(B::Transaction, H::Out)>, @@ -353,6 +351,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where R: Decode + Encode + PartialEq, NC: FnOnce() -> result::Result + UnwindSafe, { + let orig_changes = self.overlay.changes.clone(); let (result, was_native, storage_delta, changes_delta) = self.execute_aux( compute_tx, true, @@ -362,7 +361,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where if !was_native || result.is_ok() { (result, storage_delta, changes_delta) } else { - self.overlay.changes = orig_changes.clone(); + self.overlay.changes = orig_changes; let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, false, @@ -419,7 +418,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where self.execute_call_with_both_strategy( compute_tx, native_call.take(), - self.overlay.changes.clone(), on_consensus_failure, ) }, @@ -427,7 +425,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where self.execute_call_with_native_else_wasm_strategy( compute_tx, native_call.take(), - self.overlay.changes.clone(), ) }, ExecutionManager::AlwaysWasm(trust_level) => { From 32e6268128b663686789541fc50bb49b60ecc732 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Oct 2019 10:46:10 +0200 Subject: [PATCH 052/129] fix merge. --- core/state-machine/src/testing.rs | 154 ------------------------------ 1 file changed, 154 deletions(-) diff --git a/core/state-machine/src/testing.rs b/core/state-machine/src/testing.rs index a162dfc8ccceb..9e07d6c274fce 100644 --- a/core/state-machine/src/testing.rs +++ b/core/state-machine/src/testing.rs @@ -153,160 +153,6 @@ impl, N: ChangesTrieBlockNumber> From for Test } } -<<<<<<< HEAD -<<<<<<< HEAD -impl Externalities for TestExternalities where - H: Hasher, - N: ChangesTrieBlockNumber, -{ - fn storage(&self, key: &[u8]) -> Option> { - self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| - self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)) - } - - fn storage_hash(&self, key: &[u8]) -> Option { - self.storage(key).map(|v| H::hash(&v)) - } - - fn original_storage(&self, key: &[u8]) -> Option> { - self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL) - } - - fn original_storage_hash(&self, key: &[u8]) -> Option { - self.storage_hash(key) - } - - fn child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option> { - self.overlay - .child_storage(storage_key.as_ref(), key) - .map(|x| x.map(|x| x.to_vec())) - .unwrap_or_else(|| self.backend - .child_storage(storage_key.as_ref(), key) - .expect(EXT_NOT_ALLOWED_TO_FAIL) - ) - } - - fn child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option { - self.child_storage(storage_key, key).map(|v| H::hash(&v)) - } - - fn original_child_storage(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option> { - self.backend - .child_storage(storage_key.as_ref(), key) - .map(|x| x.map(|x| x.to_vec())) - .expect(EXT_NOT_ALLOWED_TO_FAIL) - } - - fn original_child_storage_hash(&self, storage_key: ChildStorageKey, key: &[u8]) -> Option { - self.child_storage_hash(storage_key, key) - } - - fn place_storage(&mut self, key: Vec, maybe_value: Option>) { - if is_child_storage_key(&key) { - panic!("Refuse to directly set child storage key"); - } - - self.overlay.set_storage(key, maybe_value); - } - - fn place_child_storage( - &mut self, - storage_key: ChildStorageKey, - key: Vec, - value: Option>, - ) { - self.overlay.set_child_storage(storage_key.into_owned(), key, value); - } - - fn kill_child_storage(&mut self, storage_key: ChildStorageKey) { - let backend = &self.backend; - let overlay = &mut self.overlay; - - overlay.clear_child_storage(storage_key.as_ref()); - backend.for_keys_in_child_storage(storage_key.as_ref(), |key| { - overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None); - }); - } - - fn clear_prefix(&mut self, prefix: &[u8]) { - if is_child_storage_key(prefix) { - panic!("Refuse to directly clear prefix that is part of child storage key"); - } - - self.overlay.clear_prefix(prefix); - - let backend = &self.backend; - let overlay = &mut self.overlay; - backend.for_keys_with_prefix(prefix, |key| { - overlay.set_storage(key.to_vec(), None); - }); - } - - fn clear_child_prefix(&mut self, storage_key: ChildStorageKey, prefix: &[u8]) { - self.overlay.clear_child_prefix(storage_key.as_ref(), prefix); - - let backend = &self.backend; - let overlay = &mut self.overlay; - backend.for_child_keys_with_prefix(storage_key.as_ref(), prefix, |key| { - overlay.set_child_storage(storage_key.as_ref().to_vec(), key.to_vec(), None); - }); - } - - fn chain_id(&self) -> u64 { 42 } - - fn storage_root(&mut self) -> H256 { - - let child_delta_iter = self.overlay.changes.owned_children_iter(); - - // compute and memoize - let delta = self.overlay.changes.iter_values(None).map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); - self.backend.full_storage_root(delta, child_delta_iter).0 - } - - fn child_storage_root(&mut self, storage_key: ChildStorageKey) -> Vec { - let storage_key = storage_key.as_ref(); - - let (root, is_empty, _) = { - let delta = self.overlay.changes.iter_values(Some(storage_key)) - .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); - - self.backend.child_storage_root(storage_key, delta) - }; - if is_empty { - self.overlay.set_storage(storage_key.into(), None); - } else { - self.overlay.set_storage(storage_key.into(), Some(root.clone())); - } - root - } - - fn storage_changes_root(&mut self, parent: H256) -> Result, ()> { - Ok(build_changes_trie::<_, _, H, N>( - &self.backend, - Some(&self.changes_trie_storage), - &self.overlay, - parent, - )?.map(|(_, root, _)| root)) - } - - fn storage_start_transaction(&mut self) { - self.overlay.start_transaction() - } - - fn storage_discard_transaction(&mut self) { - self.overlay.discard_transaction() - } - - fn storage_commit_transaction(&mut self) { - self.overlay.commit_transaction() - } - -} - -======= ->>>>>>> master -======= ->>>>>>> fc66ecdafa32965b809c1a80edee6e9416d8159d impl externalities::ExtensionStore for TestExternalities where H: Hasher, N: ChangesTrieBlockNumber, From cf943e24842be6cf2e0c1306cc0d0a7231e9be1e Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Oct 2019 10:50:17 +0200 Subject: [PATCH 053/129] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove std from include as it is default Co-Authored-By: Bastian Köcher Co-Authored-By: Sergei Pepyakin --- core/state-machine/Cargo.toml | 4 ++-- core/state-machine/fuzz/src/lib.rs | 2 -- core/state-machine/src/basic.rs | 1 - core/utils/historied-data/src/linear.rs | 2 -- srml/support/src/storage/mod.rs | 1 - 5 files changed, 2 insertions(+), 8 deletions(-) diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index 18b4fc816aeca..9a6ba6212ba67 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -15,7 +15,7 @@ trie = { package = "substrate-trie", path = "../trie" } primitives = { package = "substrate-primitives", path = "../primitives" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } codec = { package = "parity-scale-codec", version = "1.0.0" } -historied-data = { path = "../utils/historied-data", features = ["std"] } +historied-data = { path = "../utils/historied-data" } num-traits = "0.2.8" rand = "0.7.2" externalities = { package = "substrate-externalities", path = "../externalities" } @@ -24,7 +24,7 @@ externalities = { package = "substrate-externalities", path = "../externalities" hex-literal = "0.2.1" criterion = "0.2" rand = { version = "0.7.1", features = ["small_rng"] } -historied-data = { path = "../utils/historied-data", features = ["std", "test"] } +historied-data = { path = "../utils/historied-data", features = [ "test" ] } [[bench]] name = "bench" diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index 9090ef447ef08..7a5380fdc6dab 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -104,7 +104,6 @@ fn fuzz_transactions_inner(input: &[u8], check_gc: bool) { println!("input: \n {:?}", &input); } - assert!(success); } @@ -112,7 +111,6 @@ fn check_values(overlayed: &OverlayedChanges, ref_overlayed: &RefOverlayedChange let mut len = 0; let mut success = true; for (key, value) in overlayed.iter_values(None) { - let ref_value = ref_overlayed.storage(key); if Some(value) != ref_value { println!("at {:x?} different values {:x?} {:x?}", key, Some(value), ref_value); diff --git a/core/state-machine/src/basic.rs b/core/state-machine/src/basic.rs index 39ba45216a4ac..478b6e6b5ab26 100644 --- a/core/state-machine/src/basic.rs +++ b/core/state-machine/src/basic.rs @@ -225,7 +225,6 @@ impl externalities::ExtensionStore for BasicExternalities { warn!("Extensions are not supported by `BasicExternalities`."); None } - } #[cfg(test)] diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 4ebb43a7f8b99..48d2f84590f6a 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -259,7 +259,6 @@ impl States { } impl History { - /// Set a value, it uses a state history as parameter. /// This method uses `get_mut` and do remove pending /// dropped value. @@ -402,7 +401,6 @@ impl History { /// /// This method removes latest dropped values up to the latest valid value. pub fn get_mut(&mut self, history: &[TransactionState]) -> Option> { - let mut index = self.len(); if index == 0 { return None; diff --git a/srml/support/src/storage/mod.rs b/srml/support/src/storage/mod.rs index cd31308acfcb9..da6b2f5cdf685 100644 --- a/srml/support/src/storage/mod.rs +++ b/srml/support/src/storage/mod.rs @@ -28,7 +28,6 @@ pub mod child; pub mod generator; /// Execute under a transactional layer. -/// /// If the result of execution is an error, /// the transactional layer get reverted; otherwhise /// it is committed. From c71e2c50d6311605d8e5bc9b819c099c83e6d162 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Oct 2019 11:49:37 +0200 Subject: [PATCH 054/129] Smallvec is compatible with no_std. History len function public in all cases. --- core/state-machine/Cargo.toml | 3 --- core/utils/historied-data/Cargo.toml | 4 ++-- core/utils/historied-data/src/linear.rs | 15 +-------------- 3 files changed, 3 insertions(+), 19 deletions(-) diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index 9a6ba6212ba67..e4de40a627ba2 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -32,6 +32,3 @@ harness = false [features] default = [] -test = [ - "historied-data/test", -] diff --git a/core/utils/historied-data/Cargo.toml b/core/utils/historied-data/Cargo.toml index 6b5db5111c35d..1e7e4daaebea9 100644 --- a/core/utils/historied-data/Cargo.toml +++ b/core/utils/historied-data/Cargo.toml @@ -7,12 +7,12 @@ edition = "2018" [dependencies] rstd = { package = "sr-std", path = "../../sr-std", default-features = false } -smallvec = { version = "0.6", optional = true } +smallvec = { version = "0.6", default-features = false } [features] default = ["std"] std = [ "rstd/std", - "smallvec", + "smallvec/std", ] test = [] diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 48d2f84590f6a..9fcdb494a59a0 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -54,19 +54,11 @@ impl HistoriedValue { /// Array like buffer for in memory storage. /// By in memory we expect that this will /// not required persistence and is not serialized. -#[cfg(not(feature = "std"))] -type MemoryOnly = Vec>; - -/// Array like buffer for in memory storage. -/// By in memory we expect that this will -/// not required persistence and is not serialized. -#[cfg(feature = "std")] type MemoryOnly = smallvec::SmallVec<[HistoriedValue; ALLOCATED_HISTORY]>; /// Size of preallocated history per element. /// Currently at two for committed and prospective only. /// It means that using transaction in a module got a direct allocation cost. -#[cfg(feature = "std")] const ALLOCATED_HISTORY: usize = 2; /// History of value that are related to a state history (eg field `history` of @@ -101,16 +93,11 @@ impl History { history } - #[cfg(any(test, feature = "test"))] + /// Current number of inner states. pub fn len(&self) -> usize { self.0.len() } - #[cfg(not(any(test, feature = "test")))] - fn len(&self) -> usize { - self.0.len() - } - fn truncate(&mut self, index: usize) { self.0.truncate(index) } From 71c401264b9e075698019760ff6fae2e1069d061 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Oct 2019 12:53:40 +0200 Subject: [PATCH 055/129] fix fuzz, rename 'test' feature to more substratish 'test-helpers'. --- core/state-machine/Cargo.toml | 3 ++- core/state-machine/fuzz/Cargo.toml | 2 +- core/state-machine/src/overlayed_changes.rs | 4 ++-- core/utils/historied-data/Cargo.toml | 2 +- core/utils/historied-data/src/linear.rs | 14 +++++++------- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index e4de40a627ba2..98ae4be3855ff 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -24,7 +24,7 @@ externalities = { package = "substrate-externalities", path = "../externalities" hex-literal = "0.2.1" criterion = "0.2" rand = { version = "0.7.1", features = ["small_rng"] } -historied-data = { path = "../utils/historied-data", features = [ "test" ] } +historied-data = { path = "../utils/historied-data", features = [ "test-helpers" ] } [[bench]] name = "bench" @@ -32,3 +32,4 @@ harness = false [features] default = [] +test-helpers = [] diff --git a/core/state-machine/fuzz/Cargo.toml b/core/state-machine/fuzz/Cargo.toml index 90afcdf558cd1..f5c1129ab314e 100644 --- a/core/state-machine/fuzz/Cargo.toml +++ b/core/state-machine/fuzz/Cargo.toml @@ -11,7 +11,7 @@ cargo-fuzz = true [dependencies.substrate-state-machine] path = ".." -features = ["test"] +features = ["test-helpers"] [dependencies.libfuzzer-sys] git = "https://github.com/rust-fuzz/libfuzzer-sys.git" diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 17ff9bde8401a..f095dddd62947 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -536,7 +536,7 @@ impl OverlayedChanges { } } - #[cfg(any(test, feature = "test"))] + #[cfg(any(test, feature = "test-helpers"))] /// Iterator over current state of the overlay. pub fn iter_values( &self, @@ -545,7 +545,7 @@ impl OverlayedChanges { self.changes.iter_values(storage_key) } - #[cfg(any(test, feature = "test"))] + #[cfg(any(test, feature = "test-helpers"))] /// Count (slow) the number of key value, history included. /// Only for debugging or testing usage. pub fn top_count_keyvalue_pair(&self) -> usize { diff --git a/core/utils/historied-data/Cargo.toml b/core/utils/historied-data/Cargo.toml index 1e7e4daaebea9..9bff251763bc1 100644 --- a/core/utils/historied-data/Cargo.toml +++ b/core/utils/historied-data/Cargo.toml @@ -15,4 +15,4 @@ std = [ "rstd/std", "smallvec/std", ] -test = [] +test-helpers = [] diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 9fcdb494a59a0..bf5e582e99625 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -28,7 +28,7 @@ use rstd::vec; /// An entry at a given history height. #[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] pub struct HistoriedValue { /// The stored value. pub value: V, @@ -66,7 +66,7 @@ const ALLOCATED_HISTORY: usize = 2; /// /// Values are always paired with a state history index. #[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] pub struct History(MemoryOnly); impl Default for History { @@ -83,7 +83,7 @@ impl History { self.0[index].as_ref() } - #[cfg(any(test, feature = "test"))] + #[cfg(any(test, feature = "test-helpers"))] /// Create an history from an existing history. pub fn from_iter(input: impl IntoIterator>) -> Self { let mut history = History::default(); @@ -126,7 +126,7 @@ impl History { } #[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] pub struct States(Vec); impl Default for States { @@ -148,7 +148,7 @@ impl States { impl States { /// Build any state for testing only. - #[cfg(any(test, feature = "test"))] + #[cfg(any(test, feature = "test-helpers"))] pub fn test_vector(test_states: Vec) -> Self { States(test_states) } @@ -312,7 +312,7 @@ impl History { } - #[cfg(any(test, feature = "test"))] + #[cfg(any(test, feature = "test-helpers"))] pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { // index is never 0, let mut index = self.len(); @@ -335,7 +335,7 @@ impl History { None } - #[cfg(any(test, feature = "test"))] + #[cfg(any(test, feature = "test-helpers"))] pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { // index is never 0, let mut index = self.len(); From 4ca0b40b839faea5dfa32f61215c5fd8172da8ec Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 21 Oct 2019 19:32:10 +0200 Subject: [PATCH 056/129] Implement with less state. --- core/state-machine/src/changes_trie/build.rs | 6 +- core/state-machine/src/ext.rs | 1 + core/state-machine/src/lib.rs | 11 +- core/state-machine/src/overlayed_changes.rs | 40 +-- core/utils/historied-data/src/lib.rs | 19 -- core/utils/historied-data/src/linear.rs | 297 ++++++++----------- 6 files changed, 163 insertions(+), 211 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 6b924477ef702..fa1d357dd901c 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -325,8 +325,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historied_data::linear::{History, States, HistoriedValue}; - use historied_data::State as TransactionState; + use historied_data::linear::{History, States, HistoriedValue, TransactionState}; use super::*; fn prepare_for_build(zero: u64) -> ( @@ -397,7 +396,7 @@ mod test { ]); let changes = OverlayedChanges { changes: OverlayedChangeSet { - history: States::test_vector(vec![TransactionState::Committed, TransactionState::Pending]), + history: States::test_vector(vec![TransactionState::Pending, TransactionState::Pending], 1), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ (OverlayedValue { @@ -452,6 +451,7 @@ mod test { ].into_iter().collect(), }, operation_from_last_gc: 0, + not_eager_gc: false, changes_trie_config: Some(config.clone()), }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 01a2b1e97b46c..e26ba9ae29597 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -565,6 +565,7 @@ mod tests { ].into_iter().collect(), }, operation_from_last_gc: 0, + not_eager_gc: false, } } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index b30f04fd78f17..4d4335b060edf 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -311,15 +311,20 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where CallResult, ) -> CallResult { - let orig_changes = self.overlay.changes.clone(); + let orig_committed = self.overlay.changes.history.committed(); + // disable eager gc to be able to rollback + self.overlay.not_eager_gc = true; let (result, was_native, storage_delta, changes_delta) = self.execute_aux( compute_tx, true, native_call.take(), ); + self.overlay.not_eager_gc = false; if was_native { - self.overlay.changes = orig_changes; + if result.is_ok() { + self.overlay.changes.history.unchecked_rollback_committed(orig_committed); + } let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, false, @@ -351,7 +356,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where R: Decode + Encode + PartialEq, NC: FnOnce() -> result::Result + UnwindSafe, { - let orig_changes = self.overlay.changes.clone(); let (result, was_native, storage_delta, changes_delta) = self.execute_aux( compute_tx, true, @@ -361,7 +365,6 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where if !was_native || result.is_ok() { (result, storage_delta, changes_delta) } else { - self.overlay.changes = orig_changes; let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, false, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index f095dddd62947..b098d7cbf9256 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,8 +22,7 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use historied_data::linear::{States, History, HistoriedValue}; -use historied_data::State as TransactionState; +use historied_data::linear::{States, History, HistoriedValue, TransactionState}; use historied_data::DEFAULT_GC_CONF; /// The overlayed changes to state to be queried on top of the backend. @@ -42,6 +41,9 @@ pub struct OverlayedChanges { /// An additional cost for data added is added for every n bytes of data. /// n is currently defined by `DEFAULT_GC_CONF`. pub(crate) operation_from_last_gc: usize, + /// Can be set to false to ensure we keep pre committed state (in case we want to move + /// back committed cursor). + pub(crate) not_eager_gc: bool, } /// The storage value, used inside OverlayedChanges. @@ -151,18 +153,16 @@ fn set_with_extrinsic_inner_overlayed_value( impl OverlayedChangeSet { /// Garbage collect. fn gc(&mut self, eager: bool) { - let eager = if eager { - let transaction_index = self.history.transaction_indexes(); - Some(transaction_index) + let committed = if eager { + Some(self.history.committed()) } else { None }; let history = self.history.as_ref(); - let eager = || eager.as_ref().map(|t| t.as_slice()); // retain does change values - self.top.retain(|_, h_value| h_value.get_mut_pruning(history, eager()).is_some()); + self.top.retain(|_, h_value| h_value.get_mut_pruning(history, committed).is_some()); self.children.retain(|_, m| { - m.retain(|_, h_value| h_value.get_mut_pruning(history, eager()).is_some()); + m.retain(|_, h_value| h_value.get_mut_pruning(history, committed).is_some()); !m.is_empty() }); } @@ -267,8 +267,9 @@ impl OverlayedChangeSet { #[cfg(test)] pub(crate) fn top_prospective(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); + let committed = self.history.committed(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_prospective(self.history.as_ref()) { + if let Some(v) = v.get_prospective(self.history.as_ref(), committed) { result.insert(k.clone(), v.clone()); } } @@ -281,8 +282,9 @@ impl OverlayedChangeSet { #[cfg(test)] pub(crate) fn top_committed(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); + let committed = self.history.committed(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_committed(self.history.as_ref()) { + if let Some(v) = v.get_committed(self.history.as_ref(), committed) { result.insert(k.clone(), v.clone()); } } @@ -432,7 +434,7 @@ impl OverlayedChanges { self.changes.discard_prospective(); if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { self.operation_from_last_gc = 0; - self.gc(true); + self.gc(!self.not_eager_gc); } } @@ -441,7 +443,7 @@ impl OverlayedChanges { self.changes.commit_prospective(); if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { self.operation_from_last_gc = 0; - self.gc(true); + self.gc(!self.not_eager_gc); } } @@ -450,7 +452,7 @@ impl OverlayedChanges { self.changes.start_transaction(); if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { self.operation_from_last_gc = 0; - self.gc(true); + self.gc(false); } } @@ -460,7 +462,7 @@ impl OverlayedChanges { self.changes.discard_transaction(); if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { self.operation_from_last_gc = 0; - self.gc(true); + self.gc(false); } } @@ -469,7 +471,7 @@ impl OverlayedChanges { self.changes.commit_transaction(); if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { self.operation_from_last_gc = 0; - self.gc(true); + self.gc(false); } } @@ -480,15 +482,18 @@ impl OverlayedChanges { ){ let top = self.changes.top; let children = self.changes.children; + let committed = self.changes.history.committed(); let history = self.changes.history.clone(); let history2 = self.changes.history; ( top.into_iter() - .filter_map(move |(k, v)| v.into_committed(history.as_ref()).map(|v| (k, v.value))), + .filter_map(move |(k, v)| v.into_committed(history.as_ref(), committed) + .map(|v| (k, v.value))), children.into_iter().map(move |(sk, v)| { let history2 = history2.clone(); (sk, v.into_iter() - .filter_map(move |(k, v)| v.into_committed(history2.as_ref()).map(|v| (k, v.value)))) + .filter_map(move |(k, v)| v.into_committed(history2.as_ref(), committed) + .map(|v| (k, v.value)))) }) ) } @@ -513,6 +518,7 @@ impl OverlayedChanges { changes, changes_trie_config, operation_from_last_gc: 0, + not_eager_gc: false, }; committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); result.changes.commit_prospective(); diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs index 54da427320a17..afe063e52a06d 100644 --- a/core/utils/historied-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -22,25 +22,6 @@ pub mod linear; -#[derive(Debug, Clone, PartialEq, Copy)] -/// State of a data. -pub enum State { - /// Data is under change and can still be dropped. - Pending, - /// Data is under change and can still be dropped. - /// This also mark the start of a transaction. - TxPending, - /// Data is committed, but can still be dropped - /// using `discard_prospective` or `discard_transaction` - /// from a parent transaction state. - Prospective, - /// Committed is data that cannot be dropped. - Committed, - /// Data pointing to this indexed historic state should - /// not be returned and can be removed. - Dropped, -} - pub const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { trigger_transaction_gc: 100_000, trigger_commit_gc: 10_000, diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index bf5e582e99625..7c4d317376764 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -22,10 +22,22 @@ //! It only allows linear history (no branch so //! inner storage is only an array of element). -use crate::State as TransactionState; use rstd::vec::Vec; use rstd::vec; +#[derive(Debug, Clone, Eq, PartialEq)] +/// State of a transactional layer. +pub enum TransactionState { + /// Data is under change and can still be dropped. + Pending, + /// Same as pending but does count as a transaction. + TxPending, + /// Data pointing to this indexed historic state should + /// not be returned and can be removed. + Dropped, +} + + /// An entry at a given history height. #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] @@ -102,12 +114,21 @@ impl History { self.0.truncate(index) } - fn pop(&mut self) -> Option> { - self.0.pop() + fn truncate_until(&mut self, index: usize) { + if index > 0 { + if self.0.spilled() { + let owned = rstd::mem::replace(&mut self.0, Default::default()); + self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); + } else { + for i in (0..index).rev() { + self.0.remove(i); + } + } + } } - fn remove(&mut self, index: usize) { - let _ = self.0.remove(index); + fn pop(&mut self) -> Option> { + self.0.pop() } /// Append without checking if a value already exist. @@ -125,64 +146,65 @@ impl History { } + +/// States is both an indexed state to query values with history +/// and a committed index that indicates a point in time where +/// we cannot drop transaction layer. +/// Committed index is starting at 1, if it is 0 then there is no +/// committed index and all layer can be dropped. +/// There is a implicit pending state which is equal to the length +/// of this history. #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct States(Vec); +pub struct States(Vec, usize); impl Default for States { fn default() -> Self { - States(vec![TransactionState::Pending]) + States(vec![TransactionState::Pending], 0) } } impl States { + /// Get reference of state, that is enough + /// information to query or update historied + /// data. pub fn as_ref(&self) -> &[TransactionState] { self.0.as_ref() } - - pub fn iter<'a>(&'a self) -> impl Iterator + 'a { - self.0.iter().map(Clone::clone).enumerate() + /// Get index of committed layer, this is + /// additional information needed to manage + /// commit and garbage collect. + pub fn committed(&self) -> usize { + self.1 } -} -impl States { + /// Allow to rollback to a previous committed + /// index. + /// This can only work if there was no eager + /// garbage collection. + pub fn unchecked_rollback_committed(&mut self, old_committed: usize) { + self.1 = old_committed; + self.discard_prospective(); + } /// Build any state for testing only. #[cfg(any(test, feature = "test-helpers"))] - pub fn test_vector(test_states: Vec) -> Self { - States(test_states) + pub fn test_vector(test_states: Vec, committed: usize) -> Self { + States(test_states, committed) } /// Discard prospective changes to state. + /// That is revert all transaction up to the committed index. pub fn discard_prospective(&mut self) { - let mut i = self.0.len(); - while i > 0 { - i -= 1; - match self.0[i] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => self.0[i] = TransactionState::Dropped, - TransactionState::Committed => break, - } + for i in self.1 .. self.0.len() { + self.0[i] = TransactionState::Dropped; } self.0.push(TransactionState::Pending); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - debug_assert!(self.0.len() > 0); - let mut i = self.0.len(); - while i > 0 { - i -= 1; - match self.0[i] { - TransactionState::Dropped => (), - TransactionState::Prospective - | TransactionState::TxPending - | TransactionState::Pending => self.0[i] = TransactionState::Committed, - | TransactionState::Committed => break, - } - } + self.1 = self.0.len(); self.0.push(TransactionState::Pending); } @@ -195,17 +217,17 @@ impl States { /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { let mut i = self.0.len(); - while i > 0 { + while i > self.1 { i -= 1; match self.0[i] { TransactionState::Dropped => (), - TransactionState::Prospective - | TransactionState::Pending => self.0[i] = TransactionState::Dropped, + TransactionState::Pending => { + self.0[i] = TransactionState::Dropped; + }, TransactionState::TxPending => { self.0[i] = TransactionState::Dropped; break; }, - TransactionState::Committed => break, } } self.0.push(TransactionState::Pending); @@ -214,35 +236,19 @@ impl States { /// Commit a transactional layer. pub fn commit_transaction(&mut self) { let mut i = self.0.len(); - while i > 0 { + while i > self.1 { i -= 1; match self.0[i] { - TransactionState::Prospective - | TransactionState::Dropped => (), - TransactionState::Pending => self.0[i] = TransactionState::Prospective, + TransactionState::Dropped => (), + TransactionState::Pending => (), TransactionState::TxPending => { - self.0[i] = TransactionState::Prospective; + self.0[i] = TransactionState::Pending; break; }, - TransactionState::Committed => break, } } - self.0.push(TransactionState::Pending); } - /// Return array of `TxPending` indexes in state. - /// This is use as an input for garbage collection. - pub fn transaction_indexes(&self) -> Vec { - let mut transaction_index = Vec::new(); - for (i, state) in self.0.iter().enumerate() { - if &TransactionState::TxPending == state { - transaction_index.push(i); - } - } - transaction_index - } - - } impl History { @@ -278,9 +284,7 @@ impl History { match history[history_index] { TransactionState::Dropped => (), TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Committed => + | TransactionState::TxPending => return Some(value), } } @@ -300,9 +304,7 @@ impl History { match history[history_index] { TransactionState::Dropped => (), TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Committed => { + | TransactionState::TxPending => { self.truncate(index + 1); return self.pop().map(|v| v.value); }, @@ -313,31 +315,27 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] - pub fn get_prospective(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, + pub fn get_prospective(&self, history: &[TransactionState], committed: usize) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; } debug_assert!(history.len() >= index); - while index > 0 { + while index > committed { index -= 1; let HistoriedValue { value, index: history_index } = self.get_state(index); match history[history_index] { + TransactionState::Dropped => (), TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => + | TransactionState::TxPending => return Some(value), - TransactionState::Committed - | TransactionState::Dropped => (), } } None } #[cfg(any(test, feature = "test-helpers"))] - pub fn get_committed(&self, history: &[TransactionState]) -> Option<&V> { - // index is never 0, + pub fn get_committed(&self, history: &[TransactionState], committed: usize) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; @@ -346,18 +344,19 @@ impl History { while index > 0 { index -= 1; let HistoriedValue { value, index: history_index } = self.get_state(index); - match history[history_index] { - TransactionState::Committed => return Some(value), - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), + if history_index < committed { + match history[history_index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending => + return Some(value), + } } } None } - pub fn into_committed(mut self, history: &[TransactionState]) -> Option { + pub fn into_committed(mut self, history: &[TransactionState], committed: usize) -> Option { // index is never 0, let mut index = self.len(); if index == 0 { @@ -370,15 +369,15 @@ impl History { while index > 0 { index -= 1; let history_index = self.get_state(index).index; - match history[history_index] { - TransactionState::Committed => { - self.truncate(index + 1); - return self.pop().map(|v| v.value); - }, - TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective - | TransactionState::Dropped => (), + if history_index < committed { + match history[history_index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending => { + self.truncate(index + 1); + return self.pop().map(|v| v.value); + }, + } } } None @@ -400,14 +399,8 @@ impl History { index -= 1; let history_index = self.get_state(index).index; match history[history_index] { - TransactionState::Committed => { - // here we could gc all preceding values but that is additional cost - // and get_mut should stop at pending following committed. - return Some((self.mut_ref(index), history_index).into()) - }, TransactionState::Pending - | TransactionState::TxPending - | TransactionState::Prospective => { + | TransactionState::TxPending => { return Some((self.mut_ref(index), history_index).into()) }, TransactionState::Dropped => { let _ = self.pop(); }, @@ -417,83 +410,51 @@ impl History { } - /// Garbage collect the history, act as a `get_mut` with additional cost. - /// To run `eager`, a `transaction_index` parameter of all `TxPending` states - /// must be provided, then all dropped value are removed even if it means shifting - /// array byte. Otherwhise we mainly garbage collect up to last Commit state - /// (truncate left size). pub fn get_mut_pruning( &mut self, history: &[TransactionState], - transaction_index: Option<&[usize]>, - ) -> Option> { - if let Some(mut transaction_index) = transaction_index { - let mut index = self.len(); - if index == 0 { - return None; - } - // indicates that we got a value to return up to previous - // `TxPending` so values in between can be dropped. - let mut below_value = usize::max_value(); - let mut result: Option<(usize, usize)> = None; - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). - debug_assert!(history.len() >= index); - while index > 0 { - index -= 1; - let history_index = self.get_state(index).index; - match history[history_index] { - TransactionState::Committed => { - for _ in 0..index { - self.remove(0); + committed: Option, + ) -> Option> { + let mut index = self.len(); + if index == 0 { + return None; + } + let mut result = None; + let mut prune_index = self.len(); + // internal method: should be use properly + // (history of the right overlay change set + // is size aligned). + debug_assert!(history.len() >= index); + while index > 0 { + index -= 1; + let history_index = self.get_state(index).index; + match history[history_index] { + TransactionState::Pending + | TransactionState::TxPending => { + if result.is_none() { + result = Some((index, history_index)); + } + if let Some(committed) = committed { + prune_index = index; + if history_index < committed { + break; } - result = Some(result.map(|(i, history_index)| (i - index, history_index)) - .unwrap_or((0, history_index))); + } else { break; - }, - TransactionState::Pending - | TransactionState::Prospective => { - if history_index >= below_value { - self.remove(index); - result.as_mut().map(|(i, _)| *i = *i - 1); - } else { - if result.is_none() { - result = Some((index, history_index)); - } - // move to next previous `TxPending` - while below_value > history_index { - // mut slice pop - let split = transaction_index.split_last() - .map(|(v, sl)| (*v, sl)) - .unwrap_or((0, &[])); - below_value = split.0; - transaction_index = split.1; - } - } - }, - TransactionState::TxPending => { - if history_index >= below_value { - self.remove(index); - result.as_mut().map(|(i, _)| *i = *i - 1); - } else { - if result.is_none() { - result = Some((index, history_index)); - } - } - below_value = usize::max_value(); - }, - TransactionState::Dropped => { - self.remove(index); - }, - } + } + }, + TransactionState::Dropped => { let _ = self.pop(); }, } - if let Some((index, history_index)) = result { - Some((self.mut_ref(index), history_index).into()) - } else { None } - + } + if prune_index < self.len() { + self.truncate_until(prune_index); + } + // no need to clear if equal to length since we return None, meaning the value should be + // dropped. + if let Some((index, history_index)) = result { + Some((self.mut_ref(index - prune_index), history_index).into()) } else { - self.get_mut(history) + None } } From 47b8867aee22eb389b31f9113e0656227d1ce6e2 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 23 Oct 2019 10:36:46 +0200 Subject: [PATCH 057/129] revert to no gc of commited data --- Cargo.toml | 1 + core/state-machine/fuzz/src/lib.rs | 4 ++ core/utils/historied-data/src/linear.rs | 93 ++++++++++++++++++------- 3 files changed, 71 insertions(+), 27 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1a65ee1dca3de..93510e86ba92a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,7 @@ members = [ "core/sr-std", "core/sr-version", "core/state-machine", +# "core/state-machine/fuzz", "core/telemetry", "core/test-client", "core/test-runtime", diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index 7a5380fdc6dab..c483bfccdebe6 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -241,10 +241,14 @@ impl RefOverlayedChanges { #[test] fn previous_fuzzed_error() { let inputs = [ + vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, + 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, + 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], vec![50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67], vec![0x6e, 0xff, 0xf7, 0x0, 0x6e, 0xff, 0xff, 0x2d, 0xff, 0xff, 0xff, 0xe], + vec![0x2e,0x6e,0x22,0x32,0x2e,0x6e,0x22,0x32,0x3f,0x2e,], ]; for input in inputs.iter() { fuzz_transactions_inner(&input[..], true); diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 7c4d317376764..8371f4bc496f4 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -30,7 +30,7 @@ use rstd::vec; pub enum TransactionState { /// Data is under change and can still be dropped. Pending, - /// Same as pending but does count as a transaction. + /// Same as pending but does count as a transaction start. TxPending, /// Data pointing to this indexed historic state should /// not be returned and can be removed. @@ -166,11 +166,19 @@ impl Default for States { impl States { /// Get reference of state, that is enough - /// information to query or update historied + /// information to query historied /// data. pub fn as_ref(&self) -> &[TransactionState] { self.0.as_ref() } + + /// Get reference of state, that is enough + /// information to update historied + /// data. + pub fn as_ref_mut(&self) -> (&[TransactionState], usize) { + (self.0.as_ref(), self.1) + } + /// Get index of committed layer, this is /// additional information needed to manage /// commit and garbage collect. @@ -239,18 +247,37 @@ impl States { while i > self.1 { i -= 1; match self.0[i] { - TransactionState::Dropped => (), - TransactionState::Pending => (), + TransactionState::Pending + | TransactionState::Dropped => (), TransactionState::TxPending => { self.0[i] = TransactionState::Pending; break; }, } } + self.0.push(TransactionState::Pending); } } +/// Get previous index of pending state. +/// Used to say if it is possible to drop a committed transaction +/// state value. +pub fn find_previous_tx_start(states: (&[TransactionState], usize), from: usize) -> usize { + for i in (states.1 .. from).rev() { + match states.0[i] { + TransactionState::Pending + | TransactionState::TxPending => { + return i; + }, + _ => (), + } + } + 0 +} + + + impl History { /// Set a value, it uses a state history as parameter. /// This method uses `get_mut` and do remove pending @@ -395,18 +422,27 @@ impl History { // (history of the right overlay change set // is size aligned). debug_assert!(history.len() >= index); + let mut result = None; while index > 0 { index -= 1; let history_index = self.get_state(index).index; match history[history_index] { - TransactionState::Pending - | TransactionState::TxPending => { - return Some((self.mut_ref(index), history_index).into()) + TransactionState::TxPending + | TransactionState::Pending => { + result = Some((index, history_index)); + break; }, - TransactionState::Dropped => { let _ = self.pop(); }, + TransactionState::Dropped => (), } } - None + if let Some((index, history_index)) = result { + if index + 1 < self.len() { + self.truncate(index + 1); + } + Some((self.mut_ref(index), history_index).into()) + } else { + None + } } @@ -419,43 +455,46 @@ impl History { if index == 0 { return None; } - let mut result = None; let mut prune_index = self.len(); // internal method: should be use properly // (history of the right overlay change set // is size aligned). debug_assert!(history.len() >= index); + let mut result = None; while index > 0 { index -= 1; let history_index = self.get_state(index).index; match history[history_index] { - TransactionState::Pending - | TransactionState::TxPending => { - if result.is_none() { - result = Some((index, history_index)); - } - if let Some(committed) = committed { - prune_index = index; - if history_index < committed { + TransactionState::TxPending + | TransactionState::Pending => { + result = Some((index, history_index)); + if let Some(committed) = committed { + prune_index = index; + if history_index < committed { + break; + } + } else { break; } - } else { - break; - } }, - TransactionState::Dropped => { let _ = self.pop(); }, + TransactionState::Dropped => (), } } - if prune_index < self.len() { - self.truncate_until(prune_index); - } // no need to clear if equal to length since we return None, meaning the value should be // dropped. + let deleted = if prune_index < self.len() && result.is_some() { + self.truncate_until(prune_index); + prune_index + } else { + 0 + }; if let Some((index, history_index)) = result { - Some((self.mut_ref(index - prune_index), history_index).into()) + if index + 1 - deleted < self.len() { + self.truncate(index + 1 - deleted); + } + Some((self.mut_ref(index), history_index).into()) } else { None } } - } From d9a0fb501d41c728f26e77ba6646c4f5c49731c3 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 23 Oct 2019 14:33:25 +0200 Subject: [PATCH 058/129] renaming some history variables and field to states. using states with committed index for get_mut and get_mut_pruning (needed to implement merging). --- core/state-machine/src/changes_trie/build.rs | 2 +- core/state-machine/src/ext.rs | 2 +- core/state-machine/src/lib.rs | 4 +- core/state-machine/src/overlayed_changes.rs | 83 +++++++++--------- core/utils/historied-data/src/linear.rs | 89 ++++++++++---------- 5 files changed, 94 insertions(+), 86 deletions(-) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index fa1d357dd901c..85e76e76a52f8 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -396,7 +396,7 @@ mod test { ]); let changes = OverlayedChanges { changes: OverlayedChangeSet { - history: States::test_vector(vec![TransactionState::Pending, TransactionState::Pending], 1), + states: States::test_vector(vec![TransactionState::Pending, TransactionState::Pending], 1), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ (OverlayedValue { diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index e26ba9ae29597..33d0407f27752 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -547,7 +547,7 @@ mod tests { digest_levels: 0, }), changes: OverlayedChangeSet { - history: Default::default(), + states: Default::default(), children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 4d4335b060edf..69554276a86b9 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -311,7 +311,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where CallResult, ) -> CallResult { - let orig_committed = self.overlay.changes.history.committed(); + let orig_committed = self.overlay.changes.states.committed(); // disable eager gc to be able to rollback self.overlay.not_eager_gc = true; let (result, was_native, storage_delta, changes_delta) = self.execute_aux( @@ -323,7 +323,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where self.overlay.not_eager_gc = false; if was_native { if result.is_ok() { - self.overlay.changes.history.unchecked_rollback_committed(orig_committed); + self.overlay.changes.states.unchecked_rollback_committed(orig_committed); } let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index b098d7cbf9256..9be2d26570fe1 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -71,7 +71,7 @@ pub struct OverlayedValue { #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { /// Indexed state history. - pub(crate) history: States, + pub(crate) states: States, /// Top level storage changes. pub(crate) top: HashMap, History>, /// Child storage changes. @@ -96,14 +96,14 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { /// It does remove latest dropped values. fn set_with_extrinsic_overlayed_value( h_value: &mut History, - history: &[TransactionState], + states: (&[TransactionState], usize), value: Option>, extrinsic_index: Option, ) { if let Some(extrinsic) = extrinsic_index { - set_with_extrinsic_inner_overlayed_value(h_value, history, value, extrinsic) + set_with_extrinsic_inner_overlayed_value(h_value, states, value, extrinsic) } else { - h_value.set(history, OverlayedValue { + h_value.set(states, OverlayedValue { value, extrinsics: None, }) @@ -112,12 +112,12 @@ fn set_with_extrinsic_overlayed_value( fn set_with_extrinsic_inner_overlayed_value( h_value: &mut History, - history: &[TransactionState], + states: (&[TransactionState], usize), value: Option>, extrinsic_index: u32, ) { - let state = history.len() - 1; - if let Some(current) = h_value.get_mut(history) { + let state = states.0.len() - 1; + if let Some(current) = h_value.get_mut(states) { if current.index == state { current.value.value = value; current.value.extrinsics.get_or_insert_with(Default::default) @@ -154,15 +154,15 @@ impl OverlayedChangeSet { /// Garbage collect. fn gc(&mut self, eager: bool) { let committed = if eager { - Some(self.history.committed()) + Some(self.states.committed()) } else { None }; - let history = self.history.as_ref(); + let states = self.states.as_ref_mut(); // retain does change values - self.top.retain(|_, h_value| h_value.get_mut_pruning(history, committed).is_some()); + self.top.retain(|_, h_value| h_value.get_mut_pruning(states, committed).is_some()); self.children.retain(|_, m| { - m.retain(|_, h_value| h_value.get_mut_pruning(history, committed).is_some()); + m.retain(|_, h_value| h_value.get_mut_pruning(states, committed).is_some()); !m.is_empty() }); } @@ -174,28 +174,28 @@ impl OverlayedChangeSet { /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { - self.history.discard_prospective(); + self.states.discard_prospective(); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - self.history.commit_prospective(); + self.states.commit_prospective(); } /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.history.start_transaction(); + self.states.start_transaction(); } /// Discard a transactional layer. - /// A transaction is always running (history always end with pending). + /// A transaction is always running (states always end with pending). pub fn discard_transaction(&mut self) { - self.history.discard_transaction(); + self.states.discard_transaction(); } /// Commit a transactional layer. pub fn commit_transaction(&mut self) { - self.history.commit_transaction(); + self.states.commit_transaction(); } /// Iterator over current state of a given overlay, including change trie information. @@ -212,7 +212,7 @@ impl OverlayedChangeSet { .into_iter() .flat_map(move |map| map.iter() .filter_map(move |(k, v)| - v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) + v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))) ) } @@ -232,7 +232,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_ref()) + v.get(self.states.as_ref()) .map(|v| (k.as_slice(), v.value.as_ref().map(|v| v.as_slice())))) )) } @@ -245,7 +245,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.to_vec(), child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_ref()) + v.get(self.states.as_ref()) .map(|v| (k.to_vec(), v.value.as_ref().map(|v| v.to_vec())))) )) } @@ -257,7 +257,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() .filter_map(move |(k, v)| - v.get(self.history.as_ref()).map(|v| (k.as_slice(), v))) + v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))) )) } @@ -267,9 +267,9 @@ impl OverlayedChangeSet { #[cfg(test)] pub(crate) fn top_prospective(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); - let committed = self.history.committed(); + let committed = self.states.committed(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_prospective(self.history.as_ref(), committed) { + if let Some(v) = v.get_prospective(self.states.as_ref(), committed) { result.insert(k.clone(), v.clone()); } } @@ -282,9 +282,9 @@ impl OverlayedChangeSet { #[cfg(test)] pub(crate) fn top_committed(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); - let committed = self.history.committed(); + let committed = self.states.committed(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_committed(self.history.as_ref(), committed) { + if let Some(v) = v.get_committed(self.states.as_ref(), committed) { result.insert(k.clone(), v.clone()); } } @@ -327,7 +327,7 @@ impl OverlayedChanges { /// value has been set. pub fn storage(&self, key: &[u8]) -> Option> { if let Some(overlay_value) = self.changes.top.get(key) { - if let Some(o_value) = overlay_value.get(self.changes.history.as_ref()) { + if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } @@ -340,7 +340,7 @@ impl OverlayedChanges { pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.changes.children.get(storage_key) { if let Some(overlay_value) = map.get(key) { - if let Some(o_value) = overlay_value.get(self.changes.history.as_ref()) { + if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } @@ -355,7 +355,8 @@ impl OverlayedChanges { self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), value, extrinsic_index); + let states = self.changes.states.as_ref_mut(); + set_with_extrinsic_overlayed_value(entry, states, value, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -371,9 +372,10 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); + let states = self.changes.states.as_ref_mut(); set_with_extrinsic_overlayed_value( entry, - self.changes.history.as_ref(), + states, value, extrinsic_index, ); @@ -387,11 +389,12 @@ impl OverlayedChanges { /// [`discard_prospective`]: #method.discard_prospective pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) { let extrinsic_index = self.extrinsic_index(); - let history = self.changes.history.as_ref(); + let states = self.changes.states.as_ref_mut(); let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); self.operation_from_last_gc += map_entry.len(); - map_entry.values_mut().for_each(|e| set_with_extrinsic_overlayed_value(e, history, None, extrinsic_index)); + map_entry.values_mut() + .for_each(|e| set_with_extrinsic_overlayed_value(e, states, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -404,10 +407,11 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let mut number_removed = 0; + let states = self.changes.states.as_ref_mut(); for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { number_removed += 1; - set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, states, None, extrinsic_index); } } @@ -418,10 +422,11 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); if let Some(child_change) = self.changes.children.get_mut(storage_key) { let mut number_removed = 0; + let states = self.changes.states.as_ref_mut(); for (key, entry) in child_change.iter_mut() { if key.starts_with(prefix) { number_removed += 1; - set_with_extrinsic_overlayed_value(entry, self.changes.history.as_ref(), None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, states, None, extrinsic_index); } } @@ -482,17 +487,17 @@ impl OverlayedChanges { ){ let top = self.changes.top; let children = self.changes.children; - let committed = self.changes.history.committed(); - let history = self.changes.history.clone(); - let history2 = self.changes.history; + let committed = self.changes.states.committed(); + let states = self.changes.states.clone(); + let states2 = self.changes.states; ( top.into_iter() - .filter_map(move |(k, v)| v.into_committed(history.as_ref(), committed) + .filter_map(move |(k, v)| v.into_committed(states.as_ref(), committed) .map(|v| (k, v.value))), children.into_iter().map(move |(sk, v)| { - let history2 = history2.clone(); + let states2 = states2.clone(); (sk, v.into_iter() - .filter_map(move |(k, v)| v.into_committed(history2.as_ref(), committed) + .filter_map(move |(k, v)| v.into_committed(states2.as_ref(), committed) .map(|v| (k, v.value)))) }) ) diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 8371f4bc496f4..f5f1606673503 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -282,33 +282,33 @@ impl History { /// Set a value, it uses a state history as parameter. /// This method uses `get_mut` and do remove pending /// dropped value. - pub fn set(&mut self, history: &[TransactionState], value: V) { - if let Some(v) = self.get_mut(history) { - if v.index == history.len() - 1 { + pub fn set(&mut self, states: (&[TransactionState], usize), value: V) { + if let Some(v) = self.get_mut(states) { + if v.index == states.0.len() - 1 { *v.value = value; return; } } self.push_unchecked(HistoriedValue { value, - index: history.len() - 1, + index: states.0.len() - 1, }); } - /// Access to latest pending value (non dropped state in history). + /// Access to latest pending value (non dropped state). /// When possible please prefer `get_mut` as it can free /// some memory. - pub fn get(&self, history: &[TransactionState]) -> Option<&V> { + pub fn get(&self, states: &[TransactionState]) -> Option<&V> { // index is never 0, let mut index = self.len(); if index == 0 { return None; } - debug_assert!(history.len() >= index); + debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let HistoriedValue { value, index: history_index } = self.get_state(index); - match history[history_index] { + let HistoriedValue { value, index: state_index } = self.get_state(index); + match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => @@ -319,16 +319,16 @@ impl History { } /// Get latest value, consuming the historied data. - pub fn into_pending(mut self, history: &[TransactionState]) -> Option { + pub fn into_pending(mut self, states: &[TransactionState]) -> Option { let mut index = self.len(); if index == 0 { return None; } - debug_assert!(history.len() >= index); + debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let history_index = self.get_state(index).index; - match history[history_index] { + let state_index = self.get_state(index).index; + match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => { @@ -342,16 +342,16 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] - pub fn get_prospective(&self, history: &[TransactionState], committed: usize) -> Option<&V> { + pub fn get_prospective(&self, states: &[TransactionState], committed: usize) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; } - debug_assert!(history.len() >= index); + debug_assert!(states.len() >= index); while index > committed { index -= 1; - let HistoriedValue { value, index: history_index } = self.get_state(index); - match history[history_index] { + let HistoriedValue { value, index: state_index } = self.get_state(index); + match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => @@ -362,17 +362,17 @@ impl History { } #[cfg(any(test, feature = "test-helpers"))] - pub fn get_committed(&self, history: &[TransactionState], committed: usize) -> Option<&V> { + pub fn get_committed(&self, states: &[TransactionState], committed: usize) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; } - debug_assert!(history.len() >= index); + debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let HistoriedValue { value, index: history_index } = self.get_state(index); - if history_index < committed { - match history[history_index] { + let HistoriedValue { value, index: state_index } = self.get_state(index); + if state_index < committed { + match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => @@ -383,7 +383,7 @@ impl History { None } - pub fn into_committed(mut self, history: &[TransactionState], committed: usize) -> Option { + pub fn into_committed(mut self, states: &[TransactionState], committed: usize) -> Option { // index is never 0, let mut index = self.len(); if index == 0 { @@ -392,12 +392,12 @@ impl History { // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= index); + debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let history_index = self.get_state(index).index; - if history_index < committed { - match history[history_index] { + let state_index = self.get_state(index).index; + if state_index < committed { + match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => { @@ -410,10 +410,13 @@ impl History { None } - /// Access to latest pending value (non dropped state in history). + /// Access to latest pending value (non dropped state). /// /// This method removes latest dropped values up to the latest valid value. - pub fn get_mut(&mut self, history: &[TransactionState]) -> Option> { + pub fn get_mut( + &mut self, + states: (&[TransactionState], usize), + ) -> Option> { let mut index = self.len(); if index == 0 { return None; @@ -421,25 +424,25 @@ impl History { // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= index); + debug_assert!(states.0.len() >= index); let mut result = None; while index > 0 { index -= 1; - let history_index = self.get_state(index).index; - match history[history_index] { + let state_index = self.get_state(index).index; + match states.0[state_index] { TransactionState::TxPending | TransactionState::Pending => { - result = Some((index, history_index)); + result = Some((index, state_index)); break; }, TransactionState::Dropped => (), } } - if let Some((index, history_index)) = result { + if let Some((index, state_index)) = result { if index + 1 < self.len() { self.truncate(index + 1); } - Some((self.mut_ref(index), history_index).into()) + Some((self.mut_ref(index), state_index).into()) } else { None } @@ -448,7 +451,7 @@ impl History { pub fn get_mut_pruning( &mut self, - history: &[TransactionState], + states: (&[TransactionState], usize), committed: Option, ) -> Option> { let mut index = self.len(); @@ -459,18 +462,18 @@ impl History { // internal method: should be use properly // (history of the right overlay change set // is size aligned). - debug_assert!(history.len() >= index); + debug_assert!(states.0.len() >= index); let mut result = None; while index > 0 { index -= 1; - let history_index = self.get_state(index).index; - match history[history_index] { + let state_index = self.get_state(index).index; + match states.0[state_index] { TransactionState::TxPending | TransactionState::Pending => { - result = Some((index, history_index)); + result = Some((index, state_index)); if let Some(committed) = committed { prune_index = index; - if history_index < committed { + if state_index < committed { break; } } else { @@ -488,11 +491,11 @@ impl History { } else { 0 }; - if let Some((index, history_index)) = result { + if let Some((index, state_index)) = result { if index + 1 - deleted < self.len() { self.truncate(index + 1 - deleted); } - Some((self.mut_ref(index), history_index).into()) + Some((self.mut_ref(index), state_index).into()) } else { None } From 1c06ce1a1879ec4fb8bdd01fca4a33ba99da4efa Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 23 Oct 2019 16:01:08 +0200 Subject: [PATCH 059/129] satisfactory get_mut, pruning need changes. --- Cargo.lock | 24 +++++++++++++++ Cargo.toml | 1 - core/state-machine/fuzz/src/lib.rs | 1 + core/utils/historied-data/src/linear.rs | 41 ++++++++++++++++++++----- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5674fc0dde9d3..06d668e13bad5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,6 +92,11 @@ dependencies = [ "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "arbitrary" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "arbitrary" version = "0.2.0" @@ -1676,6 +1681,15 @@ name = "libc" version = "0.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "libfuzzer-sys" +version = "0.1.0" +source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#4ad88ec54fa2e7a0c877b6477cc86ea0de14a90d" +dependencies = [ + "arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "libloading" version = "0.5.2" @@ -5642,6 +5656,14 @@ dependencies = [ "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "substrate-state-machine-fuzz" +version = "0.0.1" +dependencies = [ + "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", + "substrate-state-machine 2.0.0", +] + [[package]] name = "substrate-telemetry" version = "2.0.0" @@ -6813,6 +6835,7 @@ dependencies = [ "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" "checksum app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e73a24bad9bd6a94d6395382a6c69fe071708ae4409f763c5475e14ee896313d" +"checksum arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c7d1523aa3a127adf8b27af2404c03c12825b4c4d0698f01648d63fa9df62ee" "checksum arbitrary 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" "checksum arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "bc4662175ead9cd84451d5c35070517777949a2ed84551764129cedb88384841" "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" @@ -6991,6 +7014,7 @@ dependencies = [ "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" "checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" +"checksum libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)" = "" "checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" "checksum libp2p 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4183fb4be621d97baebbbe0c499d6ae337e9e6ec955f9fa3cb29e55547dfacdb" "checksum libp2p-core 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a7ebd9d597299512e096cc1bd58e955c03ef28f33214a33b9c7e4ace109ff41" diff --git a/Cargo.toml b/Cargo.toml index 93510e86ba92a..1a65ee1dca3de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,7 +62,6 @@ members = [ "core/sr-std", "core/sr-version", "core/state-machine", -# "core/state-machine/fuzz", "core/telemetry", "core/test-client", "core/test-runtime", diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index c483bfccdebe6..72efe2c3a35d5 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -241,6 +241,7 @@ impl RefOverlayedChanges { #[test] fn previous_fuzzed_error() { let inputs = [ + vec![0xd9,0xff,0xfd,0x0,0xff,0xff,0xf8,0x1,0x92,0xff,0xbf,0x14,], vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index f5f1606673503..3bd4deebada5f 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -273,7 +273,7 @@ pub fn find_previous_tx_start(states: (&[TransactionState], usize), from: usize) _ => (), } } - 0 + states.1 } @@ -426,14 +426,32 @@ impl History { // is size aligned). debug_assert!(states.0.len() >= index); let mut result = None; + let mut previous_transaction = usize::max_value(); + let mut previous_switch = None; while index > 0 { index -= 1; let state_index = self.get_state(index).index; match states.0[state_index] { - TransactionState::TxPending - | TransactionState::Pending => { - result = Some((index, state_index)); - break; + + TransactionState::TxPending => { + if state_index >= previous_transaction { + previous_switch = Some((index, state_index)); + } else { + if result.is_none() { + result = Some((index, state_index)); + } + } + break; + }, + TransactionState::Pending => { + if state_index >= previous_transaction { + previous_switch = Some((index, state_index)); + } else { + if result.is_none() { + result = Some((index, state_index)); + previous_transaction = find_previous_tx_start(states, state_index); + } + } }, TransactionState::Dropped => (), } @@ -442,7 +460,16 @@ impl History { if index + 1 < self.len() { self.truncate(index + 1); } - Some((self.mut_ref(index), state_index).into()) + if let Some((switch_index, state_index)) = previous_switch { + if let Some(mut value) = self.pop() { + self.truncate(switch_index); + value.index = state_index; + self.push_unchecked(value); + } + Some((self.mut_ref(switch_index), state_index).into()) + } else { + Some((self.mut_ref(index), state_index).into()) + } } else { None } @@ -495,7 +522,7 @@ impl History { if index + 1 - deleted < self.len() { self.truncate(index + 1 - deleted); } - Some((self.mut_ref(index), state_index).into()) + Some((self.mut_ref(index - deleted), state_index).into()) } else { None } From c399c58146f4f6ffadaba57473195c6a3d98ad42 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 23 Oct 2019 20:31:44 +0200 Subject: [PATCH 060/129] Fix gc (fuzz seems ok), a bit more work on avoiding redundancy in linear.rs code is needed though. --- core/state-machine/fuzz/src/lib.rs | 10 ++- core/state-machine/src/overlayed_changes.rs | 11 +-- core/utils/historied-data/src/linear.rs | 82 ++++++++++++++++----- 3 files changed, 74 insertions(+), 29 deletions(-) diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index 72efe2c3a35d5..7595b4b56c25c 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -241,15 +241,17 @@ impl RefOverlayedChanges { #[test] fn previous_fuzzed_error() { let inputs = [ - vec![0xd9,0xff,0xfd,0x0,0xff,0xff,0xf8,0x1,0x92,0xff,0xbf,0x14,], - vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, - 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, - 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], + vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], + vec![0x2,0xee,0xee,0x12,0x2,0x16,0x67,0xee,0xee,0xee,], vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], vec![50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67], vec![0x6e, 0xff, 0xf7, 0x0, 0x6e, 0xff, 0xff, 0x2d, 0xff, 0xff, 0xff, 0xe], vec![0x2e,0x6e,0x22,0x32,0x2e,0x6e,0x22,0x32,0x3f,0x2e,], + vec![0xd9,0xff,0xfd,0x0,0xff,0xff,0xf8,0x1,0x92,0xff,0xbf,0x14,], + vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, + 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, + 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], ]; for input in inputs.iter() { fuzz_transactions_inner(&input[..], true); diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 9be2d26570fe1..42caf1ef8a898 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -152,17 +152,12 @@ fn set_with_extrinsic_inner_overlayed_value( impl OverlayedChangeSet { /// Garbage collect. - fn gc(&mut self, eager: bool) { - let committed = if eager { - Some(self.states.committed()) - } else { - None - }; + fn gc(&mut self, prune_commit: bool) { let states = self.states.as_ref_mut(); // retain does change values - self.top.retain(|_, h_value| h_value.get_mut_pruning(states, committed).is_some()); + self.top.retain(|_, h_value| h_value.get_mut_pruning(states, prune_commit).is_some()); self.children.retain(|_, m| { - m.retain(|_, h_value| h_value.get_mut_pruning(states, committed).is_some()); + m.retain(|_, h_value| h_value.get_mut_pruning(states, prune_commit).is_some()); !m.is_empty() }); } diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index 3bd4deebada5f..f3f4115ab5a0b 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -266,14 +266,20 @@ impl States { pub fn find_previous_tx_start(states: (&[TransactionState], usize), from: usize) -> usize { for i in (states.1 .. from).rev() { match states.0[i] { - TransactionState::Pending - | TransactionState::TxPending => { + TransactionState::TxPending => { return i; }, _ => (), } } states.1 +/* // skip dropped layer if any + for i in (0..=states.1).rev() { + if states.0[i] != TransactionState::Dropped { + return i; + } + } + states.1*/ } @@ -432,7 +438,6 @@ impl History { index -= 1; let state_index = self.get_state(index).index; match states.0[state_index] { - TransactionState::TxPending => { if state_index >= previous_transaction { previous_switch = Some((index, state_index)); @@ -450,6 +455,8 @@ impl History { if result.is_none() { result = Some((index, state_index)); previous_transaction = find_previous_tx_start(states, state_index); + } else { + break; } } }, @@ -471,6 +478,7 @@ impl History { Some((self.mut_ref(index), state_index).into()) } } else { + self.0.clear(); None } } @@ -479,40 +487,70 @@ impl History { pub fn get_mut_pruning( &mut self, states: (&[TransactionState], usize), - committed: Option, + prune_to_commit: bool, ) -> Option> { let mut index = self.len(); if index == 0 { return None; } - let mut prune_index = self.len(); + let mut prune_index = 0; // internal method: should be use properly // (history of the right overlay change set // is size aligned). debug_assert!(states.0.len() >= index); let mut result = None; + let mut previous_transaction = usize::max_value(); + let mut previous_switch = None; while index > 0 { index -= 1; let state_index = self.get_state(index).index; match states.0[state_index] { - TransactionState::TxPending - | TransactionState::Pending => { - result = Some((index, state_index)); - if let Some(committed) = committed { - prune_index = index; - if state_index < committed { + TransactionState::TxPending => { + if state_index < states.1 && index > prune_index { + prune_index = index; + } + if state_index >= previous_transaction { + previous_switch = Some((index, state_index)); + } else { + if result.is_none() { + result = Some((index, state_index)); + } + } + if prune_to_commit { + if state_index < states.1 { + break; + } + } else { + break; + } + }, + TransactionState::Pending => { + // index > prune index for only first. + if state_index < states.1 && index > prune_index { + prune_index = index; + } + + if state_index >= previous_transaction { + previous_switch = Some((index, state_index)); + } else { + if result.is_none() { + result = Some((index, state_index)); + previous_transaction = find_previous_tx_start(states, state_index); + } else { + if prune_to_commit { + if state_index < states.1 { + break; + } + } else { break; } - } else { - break; } + } }, TransactionState::Dropped => (), } } - // no need to clear if equal to length since we return None, meaning the value should be - // dropped. - let deleted = if prune_index < self.len() && result.is_some() { + let deleted = if prune_to_commit && prune_index > 0 && result.is_some() { self.truncate_until(prune_index); prune_index } else { @@ -522,8 +560,18 @@ impl History { if index + 1 - deleted < self.len() { self.truncate(index + 1 - deleted); } - Some((self.mut_ref(index - deleted), state_index).into()) + if let Some((switch_index, state_index)) = previous_switch { + if let Some(mut value) = self.pop() { + self.truncate(switch_index - deleted); + value.index = state_index; + self.push_unchecked(value); + } + Some((self.mut_ref(switch_index - deleted), state_index).into()) + } else { + Some((self.mut_ref(index - deleted), state_index).into()) + } } else { + self.0.clear(); None } } From 106d84b28660c1c05646b663a387b51f1e07fb35 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 24 Oct 2019 11:19:05 +0200 Subject: [PATCH 061/129] remove a bit of code redundancy. --- Cargo.lock | 24 ------ core/state-machine/src/overlayed_changes.rs | 6 +- core/utils/historied-data/src/lib.rs | 5 +- core/utils/historied-data/src/linear.rs | 96 +++++++-------------- 4 files changed, 36 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06d668e13bad5..5674fc0dde9d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,11 +92,6 @@ dependencies = [ "xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "arbitrary" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "arbitrary" version = "0.2.0" @@ -1681,15 +1676,6 @@ name = "libc" version = "0.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "libfuzzer-sys" -version = "0.1.0" -source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#4ad88ec54fa2e7a0c877b6477cc86ea0de14a90d" -dependencies = [ - "arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "libloading" version = "0.5.2" @@ -5656,14 +5642,6 @@ dependencies = [ "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "substrate-state-machine-fuzz" -version = "0.0.1" -dependencies = [ - "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", - "substrate-state-machine 2.0.0", -] - [[package]] name = "substrate-telemetry" version = "2.0.0" @@ -6835,7 +6813,6 @@ dependencies = [ "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" "checksum app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e73a24bad9bd6a94d6395382a6c69fe071708ae4409f763c5475e14ee896313d" -"checksum arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c7d1523aa3a127adf8b27af2404c03c12825b4c4d0698f01648d63fa9df62ee" "checksum arbitrary 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" "checksum arc-swap 0.3.11 (registry+https://github.com/rust-lang/crates.io-index)" = "bc4662175ead9cd84451d5c35070517777949a2ed84551764129cedb88384841" "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" @@ -7014,7 +6991,6 @@ dependencies = [ "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum lazycell 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" "checksum libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)" = "34fcd2c08d2f832f376f4173a231990fa5aef4e99fb569867318a227ef4c06ba" -"checksum libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)" = "" "checksum libloading 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" "checksum libp2p 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4183fb4be621d97baebbbe0c499d6ae337e9e6ec955f9fa3cb29e55547dfacdb" "checksum libp2p-core 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2a7ebd9d597299512e096cc1bd58e955c03ef28f33214a33b9c7e4ace109ff41" diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 42caf1ef8a898..623af6569d2c9 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -262,9 +262,8 @@ impl OverlayedChangeSet { #[cfg(test)] pub(crate) fn top_prospective(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); - let committed = self.states.committed(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_prospective(self.states.as_ref(), committed) { + if let Some(v) = v.get_prospective(self.states.as_ref_mut()) { result.insert(k.clone(), v.clone()); } } @@ -277,9 +276,8 @@ impl OverlayedChangeSet { #[cfg(test)] pub(crate) fn top_committed(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); - let committed = self.states.committed(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_committed(self.states.as_ref(), committed) { + if let Some(v) = v.get_committed(self.states.as_ref_mut()) { result.insert(k.clone(), v.clone()); } } diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historied-data/src/lib.rs index afe063e52a06d..2b1259eea4fa8 100644 --- a/core/utils/historied-data/src/lib.rs +++ b/core/utils/historied-data/src/lib.rs @@ -14,8 +14,9 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! History driven data storage. -//! Useful to store information with history +//! Data storage containing multiple state for a +//! key. +//! Can be use to store historical information //! on a per item basis. #![cfg_attr(not(feature = "std"), no_std)] diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historied-data/src/linear.rs index f3f4115ab5a0b..fdde5e3863599 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historied-data/src/linear.rs @@ -25,7 +25,7 @@ use rstd::vec::Vec; use rstd::vec; -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Copy)] /// State of a transactional layer. pub enum TransactionState { /// Data is under change and can still be dropped. @@ -263,6 +263,7 @@ impl States { /// Get previous index of pending state. /// Used to say if it is possible to drop a committed transaction /// state value. +/// Committed index is seen as a transaction state. pub fn find_previous_tx_start(states: (&[TransactionState], usize), from: usize) -> usize { for i in (states.1 .. from).rev() { match states.0[i] { @@ -273,13 +274,6 @@ pub fn find_previous_tx_start(states: (&[TransactionState], usize), from: usize) } } states.1 -/* // skip dropped layer if any - for i in (0..=states.1).rev() { - if states.0[i] != TransactionState::Dropped { - return i; - } - } - states.1*/ } @@ -348,16 +342,16 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] - pub fn get_prospective(&self, states: &[TransactionState], committed: usize) -> Option<&V> { + pub fn get_prospective(&self, states: (&[TransactionState], usize)) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; } - debug_assert!(states.len() >= index); - while index > committed { + debug_assert!(states.0.len() >= index); + while index > states.1 { index -= 1; let HistoriedValue { value, index: state_index } = self.get_state(index); - match states[state_index] { + match states.0[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => @@ -368,17 +362,17 @@ impl History { } #[cfg(any(test, feature = "test-helpers"))] - pub fn get_committed(&self, states: &[TransactionState], committed: usize) -> Option<&V> { + pub fn get_committed(&self, states: (&[TransactionState], usize)) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; } - debug_assert!(states.len() >= index); + debug_assert!(states.0.len() >= index); while index > 0 { index -= 1; let HistoriedValue { value, index: state_index } = self.get_state(index); - if state_index < committed { - match states[state_index] { + if state_index < states.1 { + match states.0[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => @@ -395,9 +389,6 @@ impl History { if index == 0 { return None; } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). debug_assert!(states.len() >= index); while index > 0 { index -= 1; @@ -427,9 +418,6 @@ impl History { if index == 0 { return None; } - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). debug_assert!(states.0.len() >= index); let mut result = None; let mut previous_transaction = usize::max_value(); @@ -463,27 +451,15 @@ impl History { TransactionState::Dropped => (), } } - if let Some((index, state_index)) = result { - if index + 1 < self.len() { - self.truncate(index + 1); - } - if let Some((switch_index, state_index)) = previous_switch { - if let Some(mut value) = self.pop() { - self.truncate(switch_index); - value.index = state_index; - self.push_unchecked(value); - } - Some((self.mut_ref(switch_index), state_index).into()) - } else { - Some((self.mut_ref(index), state_index).into()) - } - } else { - self.0.clear(); - None - } + self.clear_terminal_values(result, previous_switch, 0) } + /// Method to prune regarding a full state. + /// It also returns the last value as mutable. + /// Internally it acts like `get_mut` with an + /// additional cleaning capacity to clean committed + /// state if `prune_to_commit` is set to true. pub fn get_mut_pruning( &mut self, states: (&[TransactionState], usize), @@ -494,9 +470,6 @@ impl History { return None; } let mut prune_index = 0; - // internal method: should be use properly - // (history of the right overlay change set - // is size aligned). debug_assert!(states.0.len() >= index); let mut result = None; let mut previous_transaction = usize::max_value(); @@ -505,27 +478,8 @@ impl History { index -= 1; let state_index = self.get_state(index).index; match states.0[state_index] { - TransactionState::TxPending => { - if state_index < states.1 && index > prune_index { - prune_index = index; - } - if state_index >= previous_transaction { - previous_switch = Some((index, state_index)); - } else { - if result.is_none() { - result = Some((index, state_index)); - } - } - if prune_to_commit { - if state_index < states.1 { - break; - } - } else { - break; - } - }, - TransactionState::Pending => { - // index > prune index for only first. + state @ TransactionState::TxPending + | state @ TransactionState::Pending => { if state_index < states.1 && index > prune_index { prune_index = index; } @@ -535,7 +489,9 @@ impl History { } else { if result.is_none() { result = Some((index, state_index)); - previous_transaction = find_previous_tx_start(states, state_index); + if state == TransactionState::Pending { + previous_transaction = find_previous_tx_start(states, state_index); + } } else { if prune_to_commit { if state_index < states.1 { @@ -556,6 +512,16 @@ impl History { } else { 0 }; + self.clear_terminal_values(result, previous_switch, deleted) + } + + #[inline] + fn clear_terminal_values( + &mut self, + result: Option<(usize, usize)>, + previous_switch: Option<(usize, usize)>, + deleted: usize, + ) -> Option> { if let Some((index, state_index)) = result { if index + 1 - deleted < self.len() { self.truncate(index + 1 - deleted); From 6b5aeeaf0df70fc345cb2734b878b2e6067dd639 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 24 Oct 2019 11:53:40 +0200 Subject: [PATCH 062/129] rename historied to historical --- Cargo.lock | 6 +-- Cargo.toml | 2 +- core/state-machine/Cargo.toml | 4 +- core/state-machine/src/changes_trie/build.rs | 16 +++---- core/state-machine/src/ext.rs | 6 +-- core/state-machine/src/overlayed_changes.rs | 10 ++--- core/test-runtime/Cargo.toml | 4 +- core/test-runtime/src/lib.rs | 14 +++--- .../Cargo.toml | 2 +- .../README.md | 4 +- .../src/lib.rs | 0 .../src/linear.rs | 44 +++++++++---------- 12 files changed, 56 insertions(+), 56 deletions(-) rename core/utils/{historied-data => historical-data}/Cargo.toml (93%) rename core/utils/{historied-data => historical-data}/README.md (84%) rename core/utils/{historied-data => historical-data}/src/lib.rs (100%) rename core/utils/{historied-data => historical-data}/src/linear.rs (92%) diff --git a/Cargo.lock b/Cargo.lock index 5674fc0dde9d3..604775a1c979f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1241,7 +1241,7 @@ dependencies = [ ] [[package]] -name = "historied-data" +name = "historical-data" version = "2.0.0" dependencies = [ "smallvec 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5628,7 +5628,7 @@ dependencies = [ "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "historied-data 2.0.0", + "historical-data 2.0.0", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -5685,7 +5685,7 @@ name = "substrate-test-runtime" version = "2.0.0" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "historied-data 2.0.0", + "historical-data 2.0.0", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 1a65ee1dca3de..0bced7ab7e446 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ members = [ "core/transaction-pool/graph", "core/trie", "core/utils/fork-tree", - "core/utils/historied-data", + "core/utils/historical-data", "core/utils/wasm-builder", "core/utils/wasm-builder-runner", "core/wasm-interface", diff --git a/core/state-machine/Cargo.toml b/core/state-machine/Cargo.toml index 98ae4be3855ff..a0c1c429b342a 100644 --- a/core/state-machine/Cargo.toml +++ b/core/state-machine/Cargo.toml @@ -15,7 +15,7 @@ trie = { package = "substrate-trie", path = "../trie" } primitives = { package = "substrate-primitives", path = "../primitives" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } codec = { package = "parity-scale-codec", version = "1.0.0" } -historied-data = { path = "../utils/historied-data" } +historical-data = { path = "../utils/historical-data" } num-traits = "0.2.8" rand = "0.7.2" externalities = { package = "substrate-externalities", path = "../externalities" } @@ -24,7 +24,7 @@ externalities = { package = "substrate-externalities", path = "../externalities" hex-literal = "0.2.1" criterion = "0.2" rand = { version = "0.7.1", features = ["small_rng"] } -historied-data = { path = "../utils/historied-data", features = [ "test-helpers" ] } +historical-data = { path = "../utils/historical-data", features = [ "test-helpers" ] } [[bench]] name = "bench" diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 85e76e76a52f8..e671bc30ed686 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -325,7 +325,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historied_data::linear::{History, States, HistoriedValue, TransactionState}; + use historical_data::linear::{History, States, HistoricalValue, TransactionState}; use super::*; fn prepare_for_build(zero: u64) -> ( @@ -403,7 +403,7 @@ mod test { value: Some(3u32.encode()), extrinsics: None, }, 0), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![100], History::from_iter(vec![ (OverlayedValue { value: Some(vec![202]), @@ -413,19 +413,19 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, 1), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![101], History::from_iter(vec![ (OverlayedValue { value: Some(vec![203]), extrinsics: Some(vec![1].into_iter().collect()) }, 0), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![103], History::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![0, 1].into_iter().collect()) }, 1), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect(), children: vec![ (child_trie_key1, vec![ @@ -438,7 +438,7 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, 1), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect()), (child_trie_key2, vec![ (vec![100], History::from_iter(vec![ @@ -446,7 +446,7 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) }, 1), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect()), ].into_iter().collect(), }, @@ -649,7 +649,7 @@ mod test { value: None, extrinsics: Some(vec![1].into_iter().collect()), }, 1), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))); + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))); let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; let changes_trie_nodes = prepare_input( diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 33d0407f27752..21613c9db04a2 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -533,7 +533,7 @@ mod tests { use crate::changes_trie::{Configuration as ChangesTrieConfiguration, InMemoryStorage as InMemoryChangesTrieStorage}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historied_data::linear::{History, HistoriedValue}; + use historical_data::linear::{History, HistoricalValue}; type TestBackend = InMemory; type TestChangesTrieStorage = InMemoryChangesTrieStorage; @@ -555,13 +555,13 @@ mod tests { value: Some(3u32.encode()), extrinsics: Some(vec![1].into_iter().collect()) }, 0), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![1], History::from_iter(vec![ (OverlayedValue { value: Some(vec![100]), extrinsics: Some(vec![1].into_iter().collect()) }, 0), - ].into_iter().map(|(value, index)| HistoriedValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect(), }, operation_from_last_gc: 0, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 623af6569d2c9..32532eaab3918 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,8 +22,8 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use historied_data::linear::{States, History, HistoriedValue, TransactionState}; -use historied_data::DEFAULT_GC_CONF; +use historical_data::linear::{States, History, HistoricalValue, TransactionState}; +use historical_data::DEFAULT_GC_CONF; /// The overlayed changes to state to be queried on top of the backend. /// @@ -85,7 +85,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = History::default(); - history.push_unchecked(HistoriedValue { value, index: 0 }); + history.push_unchecked(HistoricalValue { value, index: 0 }); history })).collect(); result @@ -126,7 +126,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.value.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.push_unchecked(HistoriedValue { + h_value.push_unchecked(HistoricalValue { value: OverlayedValue { value, extrinsics, @@ -139,7 +139,7 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.push_unchecked(HistoriedValue { + h_value.push_unchecked(HistoricalValue { value: OverlayedValue { value, extrinsics, diff --git a/core/test-runtime/Cargo.toml b/core/test-runtime/Cargo.toml index f7cedd021cf41..08343251fa70b 100644 --- a/core/test-runtime/Cargo.toml +++ b/core/test-runtime/Cargo.toml @@ -32,7 +32,7 @@ srml-babe = { path = "../../srml/babe", default-features = false } srml-timestamp = { path = "../../srml/timestamp", default-features = false } srml-system = { path = "../../srml/system", default-features = false } srml-system-rpc-runtime-api = { path = "../../srml/system/rpc/runtime-api", default-features = false } -historied-data = { path = "../utils/historied-data", default-features = false } +historical-data = { path = "../utils/historical-data", default-features = false } [dev-dependencies] substrate-executor = { path = "../executor" } @@ -73,5 +73,5 @@ std = [ "srml-system-rpc-runtime-api/std", "app-crypto/std", "session/std", - "historied-data/std", + "historical-data/std", ] diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 13b30730b4e2e..d02d211bff3c9 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -574,7 +574,7 @@ cfg_if! { } fn use_history_data() { - test_historied_data() + test_historical_data() } fn use_transactions() -> u64 { @@ -771,7 +771,7 @@ cfg_if! { } fn use_history_data() { - test_historied_data() + test_historical_data() } fn use_transactions() -> u64 { @@ -927,19 +927,19 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { (signature, public0) } -fn test_historied_data() { - let mut states = historied_data::linear::States::default(); - let mut value = historied_data::linear::History::default(); +fn test_historical_data() { + let mut states = historical_data::linear::States::default(); + let mut value = historical_data::linear::History::default(); if value.get(states.as_ref()) != None { panic!("Got a value for empty data"); } - value.set(states.as_ref(), 42u64); + value.set(states.as_ref_mut(), 42u64); states.start_transaction(); if value.get(states.as_ref()) != Some(&42) { panic!("Got a wrong result accessing a one element data"); } - value.set(states.as_ref(), 43u64); + value.set(states.as_ref_mut(), 43u64); if value.get(states.as_ref()) != Some(&43) { panic!("Got a wrong result accessing a two element data"); } diff --git a/core/utils/historied-data/Cargo.toml b/core/utils/historical-data/Cargo.toml similarity index 93% rename from core/utils/historied-data/Cargo.toml rename to core/utils/historical-data/Cargo.toml index 9bff251763bc1..91103903084d7 100644 --- a/core/utils/historied-data/Cargo.toml +++ b/core/utils/historical-data/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "historied-data" +name = "historical-data" version = "2.0.0" authors = ["Parity Technologies "] description = "Data associated with its history" diff --git a/core/utils/historied-data/README.md b/core/utils/historical-data/README.md similarity index 84% rename from core/utils/historied-data/README.md rename to core/utils/historical-data/README.md index b8ef518428c5d..e559410b5dd41 100644 --- a/core/utils/historied-data/README.md +++ b/core/utils/historical-data/README.md @@ -1,4 +1,4 @@ -## Historied data +## Historical data Crate with methods to manage data that stores its own history. @@ -13,6 +13,6 @@ History is serialize as a per item basis. This crates is `no_std` compatible as long as the `std` feature is not enabled. -For more information see +For more information see License: GPL-3.0 diff --git a/core/utils/historied-data/src/lib.rs b/core/utils/historical-data/src/lib.rs similarity index 100% rename from core/utils/historied-data/src/lib.rs rename to core/utils/historical-data/src/lib.rs diff --git a/core/utils/historied-data/src/linear.rs b/core/utils/historical-data/src/linear.rs similarity index 92% rename from core/utils/historied-data/src/linear.rs rename to core/utils/historical-data/src/linear.rs index fdde5e3863599..5cb2b9011dd6f 100644 --- a/core/utils/historied-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -41,22 +41,22 @@ pub enum TransactionState { /// An entry at a given history height. #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct HistoriedValue { +pub struct HistoricalValue { /// The stored value. pub value: V, /// The moment in history when the value got set. pub index: usize, } -impl From<(V, usize)> for HistoriedValue { - fn from(input: (V, usize)) -> HistoriedValue { - HistoriedValue { value: input.0, index: input.1 } +impl From<(V, usize)> for HistoricalValue { + fn from(input: (V, usize)) -> HistoricalValue { + HistoricalValue { value: input.0, index: input.1 } } } -impl HistoriedValue { - fn as_ref(&self) -> HistoriedValue<&V> { - HistoriedValue { +impl HistoricalValue { + fn as_ref(&self) -> HistoricalValue<&V> { + HistoricalValue { value: &self.value, index: self.index, } @@ -66,7 +66,7 @@ impl HistoriedValue { /// Array like buffer for in memory storage. /// By in memory we expect that this will /// not required persistence and is not serialized. -type MemoryOnly = smallvec::SmallVec<[HistoriedValue; ALLOCATED_HISTORY]>; +type MemoryOnly = smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>; /// Size of preallocated history per element. /// Currently at two for committed and prospective only. @@ -91,13 +91,13 @@ impl Default for History { // buffer specific functions. impl History { - fn get_state(&self, index: usize) -> HistoriedValue<&V> { + fn get_state(&self, index: usize) -> HistoricalValue<&V> { self.0[index].as_ref() } #[cfg(any(test, feature = "test-helpers"))] /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator>) -> Self { + pub fn from_iter(input: impl IntoIterator>) -> Self { let mut history = History::default(); for v in input { history.push_unchecked(v); @@ -127,7 +127,7 @@ impl History { } } - fn pop(&mut self) -> Option> { + fn pop(&mut self) -> Option> { self.0.pop() } @@ -136,7 +136,7 @@ impl History { /// This method shall only be call after a `get_mut` where /// the returned index indicate that a `set` will result /// in appending a value. - pub fn push_unchecked(&mut self, value: HistoriedValue) { + pub fn push_unchecked(&mut self, value: HistoricalValue) { self.0.push(value) } @@ -166,14 +166,14 @@ impl Default for States { impl States { /// Get reference of state, that is enough - /// information to query historied + /// information to query historical /// data. pub fn as_ref(&self) -> &[TransactionState] { self.0.as_ref() } /// Get reference of state, that is enough - /// information to update historied + /// information to update historical /// data. pub fn as_ref_mut(&self) -> (&[TransactionState], usize) { (self.0.as_ref(), self.1) @@ -289,7 +289,7 @@ impl History { return; } } - self.push_unchecked(HistoriedValue { + self.push_unchecked(HistoricalValue { value, index: states.0.len() - 1, }); @@ -307,7 +307,7 @@ impl History { debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let HistoriedValue { value, index: state_index } = self.get_state(index); + let HistoricalValue { value, index: state_index } = self.get_state(index); match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending @@ -318,7 +318,7 @@ impl History { None } - /// Get latest value, consuming the historied data. + /// Get latest value, consuming the historical data. pub fn into_pending(mut self, states: &[TransactionState]) -> Option { let mut index = self.len(); if index == 0 { @@ -350,7 +350,7 @@ impl History { debug_assert!(states.0.len() >= index); while index > states.1 { index -= 1; - let HistoriedValue { value, index: state_index } = self.get_state(index); + let HistoricalValue { value, index: state_index } = self.get_state(index); match states.0[state_index] { TransactionState::Dropped => (), TransactionState::Pending @@ -370,7 +370,7 @@ impl History { debug_assert!(states.0.len() >= index); while index > 0 { index -= 1; - let HistoriedValue { value, index: state_index } = self.get_state(index); + let HistoricalValue { value, index: state_index } = self.get_state(index); if state_index < states.1 { match states.0[state_index] { TransactionState::Dropped => (), @@ -413,7 +413,7 @@ impl History { pub fn get_mut( &mut self, states: (&[TransactionState], usize), - ) -> Option> { + ) -> Option> { let mut index = self.len(); if index == 0 { return None; @@ -464,7 +464,7 @@ impl History { &mut self, states: (&[TransactionState], usize), prune_to_commit: bool, - ) -> Option> { + ) -> Option> { let mut index = self.len(); if index == 0 { return None; @@ -521,7 +521,7 @@ impl History { result: Option<(usize, usize)>, previous_switch: Option<(usize, usize)>, deleted: usize, - ) -> Option> { + ) -> Option> { if let Some((index, state_index)) = result { if index + 1 - deleted < self.len() { self.truncate(index + 1 - deleted); From 594ce640b5c04fc0481f866448e8772a183cb72a Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 24 Oct 2019 14:24:59 +0200 Subject: [PATCH 063/129] removing dubious as_ref_mut method (intermediate tuple is not optimized very well). --- core/state-machine/src/overlayed_changes.rs | 28 +++++++++------------ core/test-runtime/src/lib.rs | 4 +-- core/utils/historical-data/src/linear.rs | 20 +++++++-------- 3 files changed, 23 insertions(+), 29 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 32532eaab3918..fe3595b06c1fb 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,7 +22,7 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use historical_data::linear::{States, History, HistoricalValue, TransactionState}; +use historical_data::linear::{States, History, HistoricalValue}; use historical_data::DEFAULT_GC_CONF; /// The overlayed changes to state to be queried on top of the backend. @@ -96,7 +96,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { /// It does remove latest dropped values. fn set_with_extrinsic_overlayed_value( h_value: &mut History, - states: (&[TransactionState], usize), + states: &States, value: Option>, extrinsic_index: Option, ) { @@ -112,11 +112,11 @@ fn set_with_extrinsic_overlayed_value( fn set_with_extrinsic_inner_overlayed_value( h_value: &mut History, - states: (&[TransactionState], usize), + states: &States, value: Option>, extrinsic_index: u32, ) { - let state = states.0.len() - 1; + let state = states.len() - 1; if let Some(current) = h_value.get_mut(states) { if current.index == state { current.value.value = value; @@ -153,7 +153,7 @@ fn set_with_extrinsic_inner_overlayed_value( impl OverlayedChangeSet { /// Garbage collect. fn gc(&mut self, prune_commit: bool) { - let states = self.states.as_ref_mut(); + let states = &self.states; // retain does change values self.top.retain(|_, h_value| h_value.get_mut_pruning(states, prune_commit).is_some()); self.children.retain(|_, m| { @@ -263,7 +263,7 @@ impl OverlayedChangeSet { pub(crate) fn top_prospective(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_prospective(self.states.as_ref_mut()) { + if let Some(v) = v.get_prospective(&self.states) { result.insert(k.clone(), v.clone()); } } @@ -277,7 +277,7 @@ impl OverlayedChangeSet { pub(crate) fn top_committed(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_committed(self.states.as_ref_mut()) { + if let Some(v) = v.get_committed(&self.states) { result.insert(k.clone(), v.clone()); } } @@ -348,8 +348,7 @@ impl OverlayedChanges { self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - let states = self.changes.states.as_ref_mut(); - set_with_extrinsic_overlayed_value(entry, states, value, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, &self.changes.states, value, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -365,10 +364,9 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); - let states = self.changes.states.as_ref_mut(); set_with_extrinsic_overlayed_value( entry, - states, + &self.changes.states, value, extrinsic_index, ); @@ -382,7 +380,7 @@ impl OverlayedChanges { /// [`discard_prospective`]: #method.discard_prospective pub(crate) fn clear_child_storage(&mut self, storage_key: &[u8]) { let extrinsic_index = self.extrinsic_index(); - let states = self.changes.states.as_ref_mut(); + let states = &self.changes.states; let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); self.operation_from_last_gc += map_entry.len(); @@ -400,11 +398,10 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); let mut number_removed = 0; - let states = self.changes.states.as_ref_mut(); for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { number_removed += 1; - set_with_extrinsic_overlayed_value(entry, states, None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } @@ -415,11 +412,10 @@ impl OverlayedChanges { let extrinsic_index = self.extrinsic_index(); if let Some(child_change) = self.changes.children.get_mut(storage_key) { let mut number_removed = 0; - let states = self.changes.states.as_ref_mut(); for (key, entry) in child_change.iter_mut() { if key.starts_with(prefix) { number_removed += 1; - set_with_extrinsic_overlayed_value(entry, states, None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index d02d211bff3c9..d05a3116bc0e0 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -934,12 +934,12 @@ fn test_historical_data() { panic!("Got a value for empty data"); } - value.set(states.as_ref_mut(), 42u64); + value.set(&states, 42u64); states.start_transaction(); if value.get(states.as_ref()) != Some(&42) { panic!("Got a wrong result accessing a one element data"); } - value.set(states.as_ref_mut(), 43u64); + value.set(&states, 43u64); if value.get(states.as_ref()) != Some(&43) { panic!("Got a wrong result accessing a two element data"); } diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs index 5cb2b9011dd6f..93741b327f384 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -172,11 +172,9 @@ impl States { self.0.as_ref() } - /// Get reference of state, that is enough - /// information to update historical - /// data. - pub fn as_ref_mut(&self) -> (&[TransactionState], usize) { - (self.0.as_ref(), self.1) + /// Current number of inner states. + pub fn len(&self) -> usize { + self.0.len() } /// Get index of committed layer, this is @@ -264,7 +262,7 @@ impl States { /// Used to say if it is possible to drop a committed transaction /// state value. /// Committed index is seen as a transaction state. -pub fn find_previous_tx_start(states: (&[TransactionState], usize), from: usize) -> usize { +pub fn find_previous_tx_start(states: &States, from: usize) -> usize { for i in (states.1 .. from).rev() { match states.0[i] { TransactionState::TxPending => { @@ -282,7 +280,7 @@ impl History { /// Set a value, it uses a state history as parameter. /// This method uses `get_mut` and do remove pending /// dropped value. - pub fn set(&mut self, states: (&[TransactionState], usize), value: V) { + pub fn set(&mut self, states: &States, value: V) { if let Some(v) = self.get_mut(states) { if v.index == states.0.len() - 1 { *v.value = value; @@ -342,7 +340,7 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] - pub fn get_prospective(&self, states: (&[TransactionState], usize)) -> Option<&V> { + pub fn get_prospective(&self, states: &States) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; @@ -362,7 +360,7 @@ impl History { } #[cfg(any(test, feature = "test-helpers"))] - pub fn get_committed(&self, states: (&[TransactionState], usize)) -> Option<&V> { + pub fn get_committed(&self, states: &States) -> Option<&V> { let mut index = self.len(); if index == 0 { return None; @@ -412,7 +410,7 @@ impl History { /// This method removes latest dropped values up to the latest valid value. pub fn get_mut( &mut self, - states: (&[TransactionState], usize), + states: &States, ) -> Option> { let mut index = self.len(); if index == 0 { @@ -462,7 +460,7 @@ impl History { /// state if `prune_to_commit` is set to true. pub fn get_mut_pruning( &mut self, - states: (&[TransactionState], usize), + states: &States, prune_to_commit: bool, ) -> Option> { let mut index = self.len(); From 7546e6a55e0d9159f94e777e5e4a99ed4792b523 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 24 Oct 2019 14:35:39 +0200 Subject: [PATCH 064/129] Named field for States. --- core/utils/historical-data/src/linear.rs | 90 +++++++++++++----------- 1 file changed, 48 insertions(+), 42 deletions(-) diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs index 93741b327f384..8fdb8d52bd1f2 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -156,11 +156,17 @@ impl History { /// of this history. #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct States(Vec, usize); +pub struct States { + history: Vec, + commit: usize, +} impl Default for States { fn default() -> Self { - States(vec![TransactionState::Pending], 0) + States { + history: vec![TransactionState::Pending], + commit: 0, + } } } @@ -169,19 +175,19 @@ impl States { /// information to query historical /// data. pub fn as_ref(&self) -> &[TransactionState] { - self.0.as_ref() + self.history.as_ref() } /// Current number of inner states. pub fn len(&self) -> usize { - self.0.len() + self.history.len() } /// Get index of committed layer, this is /// additional information needed to manage /// commit and garbage collect. pub fn committed(&self) -> usize { - self.1 + self.commit } /// Allow to rollback to a previous committed @@ -189,71 +195,71 @@ impl States { /// This can only work if there was no eager /// garbage collection. pub fn unchecked_rollback_committed(&mut self, old_committed: usize) { - self.1 = old_committed; + self.commit = old_committed; self.discard_prospective(); } /// Build any state for testing only. #[cfg(any(test, feature = "test-helpers"))] - pub fn test_vector(test_states: Vec, committed: usize) -> Self { - States(test_states, committed) + pub fn test_vector(history: Vec, commit: usize) -> Self { + States { history, commit } } /// Discard prospective changes to state. /// That is revert all transaction up to the committed index. pub fn discard_prospective(&mut self) { - for i in self.1 .. self.0.len() { - self.0[i] = TransactionState::Dropped; + for i in self.commit .. self.history.len() { + self.history[i] = TransactionState::Dropped; } - self.0.push(TransactionState::Pending); + self.history.push(TransactionState::Pending); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - self.1 = self.0.len(); - self.0.push(TransactionState::Pending); + self.commit = self.history.len(); + self.history.push(TransactionState::Pending); } /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.0.push(TransactionState::TxPending); + self.history.push(TransactionState::TxPending); } /// Discard a transactional layer. /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { - let mut i = self.0.len(); - while i > self.1 { + let mut i = self.history.len(); + while i > self.commit { i -= 1; - match self.0[i] { + match self.history[i] { TransactionState::Dropped => (), TransactionState::Pending => { - self.0[i] = TransactionState::Dropped; + self.history[i] = TransactionState::Dropped; }, TransactionState::TxPending => { - self.0[i] = TransactionState::Dropped; + self.history[i] = TransactionState::Dropped; break; }, } } - self.0.push(TransactionState::Pending); + self.history.push(TransactionState::Pending); } /// Commit a transactional layer. pub fn commit_transaction(&mut self) { - let mut i = self.0.len(); - while i > self.1 { + let mut i = self.history.len(); + while i > self.commit { i -= 1; - match self.0[i] { + match self.history[i] { TransactionState::Pending | TransactionState::Dropped => (), TransactionState::TxPending => { - self.0[i] = TransactionState::Pending; + self.history[i] = TransactionState::Pending; break; }, } } - self.0.push(TransactionState::Pending); + self.history.push(TransactionState::Pending); } } @@ -263,15 +269,15 @@ impl States { /// state value. /// Committed index is seen as a transaction state. pub fn find_previous_tx_start(states: &States, from: usize) -> usize { - for i in (states.1 .. from).rev() { - match states.0[i] { + for i in (states.commit .. from).rev() { + match states.history[i] { TransactionState::TxPending => { return i; }, _ => (), } } - states.1 + states.commit } @@ -282,14 +288,14 @@ impl History { /// dropped value. pub fn set(&mut self, states: &States, value: V) { if let Some(v) = self.get_mut(states) { - if v.index == states.0.len() - 1 { + if v.index == states.history.len() - 1 { *v.value = value; return; } } self.push_unchecked(HistoricalValue { value, - index: states.0.len() - 1, + index: states.history.len() - 1, }); } @@ -345,11 +351,11 @@ impl History { if index == 0 { return None; } - debug_assert!(states.0.len() >= index); - while index > states.1 { + debug_assert!(states.history.len() >= index); + while index > states.commit { index -= 1; let HistoricalValue { value, index: state_index } = self.get_state(index); - match states.0[state_index] { + match states.history[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => @@ -365,12 +371,12 @@ impl History { if index == 0 { return None; } - debug_assert!(states.0.len() >= index); + debug_assert!(states.history.len() >= index); while index > 0 { index -= 1; let HistoricalValue { value, index: state_index } = self.get_state(index); - if state_index < states.1 { - match states.0[state_index] { + if state_index < states.commit { + match states.history[state_index] { TransactionState::Dropped => (), TransactionState::Pending | TransactionState::TxPending => @@ -416,14 +422,14 @@ impl History { if index == 0 { return None; } - debug_assert!(states.0.len() >= index); + debug_assert!(states.history.len() >= index); let mut result = None; let mut previous_transaction = usize::max_value(); let mut previous_switch = None; while index > 0 { index -= 1; let state_index = self.get_state(index).index; - match states.0[state_index] { + match states.history[state_index] { TransactionState::TxPending => { if state_index >= previous_transaction { previous_switch = Some((index, state_index)); @@ -468,17 +474,17 @@ impl History { return None; } let mut prune_index = 0; - debug_assert!(states.0.len() >= index); + debug_assert!(states.history.len() >= index); let mut result = None; let mut previous_transaction = usize::max_value(); let mut previous_switch = None; while index > 0 { index -= 1; let state_index = self.get_state(index).index; - match states.0[state_index] { + match states.history[state_index] { state @ TransactionState::TxPending | state @ TransactionState::Pending => { - if state_index < states.1 && index > prune_index { + if state_index < states.commit && index > prune_index { prune_index = index; } @@ -492,7 +498,7 @@ impl History { } } else { if prune_to_commit { - if state_index < states.1 { + if state_index < states.commit { break; } } else { From 2e6fa4c1bcad7425aa8723f8488f3961970c4be9 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 24 Oct 2019 15:42:19 +0200 Subject: [PATCH 065/129] change find_previous_transaction to use an in memory state. Gain are non existent on bench (not may layer). --- core/state-machine/fuzz/src/lib.rs | 1 + core/state-machine/src/changes_trie/build.rs | 6 ++- core/utils/historical-data/src/linear.rs | 52 +++++++++++++++----- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index 7595b4b56c25c..f8f98bbe6ae13 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -241,6 +241,7 @@ impl RefOverlayedChanges { #[test] fn previous_fuzzed_error() { let inputs = [ + vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], vec![0x2,0xee,0xee,0x12,0x2,0x16,0x67,0xee,0xee,0xee,], vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index e671bc30ed686..c8af76ec8a7a9 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -396,7 +396,11 @@ mod test { ]); let changes = OverlayedChanges { changes: OverlayedChangeSet { - states: States::test_vector(vec![TransactionState::Pending, TransactionState::Pending], 1), + states: States::test_vector( + vec![TransactionState::Pending, TransactionState::Pending], + vec![0, 0], + 1, + ), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ (OverlayedValue { diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs index 8fdb8d52bd1f2..5a0b76b870633 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -158,6 +158,23 @@ impl History { #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] pub struct States { history: Vec, + // keep trace of previous transaction. + // associatied logic is that of: + // ``` + // use historical_data::linear::States; + // fn find_previous_tx_start(states: &States, from: usize) -> usize { + // for i in (states.commit .. from).rev() { + // match states.history[i] { + // TransactionState::TxPending => { + // return i; + // }, + // _ => (), + // } + // } + // states.commit + // } + // ``` + previous_transaction: Vec, commit: usize, } @@ -165,6 +182,7 @@ impl Default for States { fn default() -> Self { States { history: vec![TransactionState::Pending], + previous_transaction: vec![0], commit: 0, } } @@ -201,8 +219,12 @@ impl States { /// Build any state for testing only. #[cfg(any(test, feature = "test-helpers"))] - pub fn test_vector(history: Vec, commit: usize) -> Self { - States { history, commit } + pub fn test_vector( + history: Vec, + previous_transaction: Vec, + commit: usize, + ) -> Self { + States { history, previous_transaction, commit } } /// Discard prospective changes to state. @@ -210,18 +232,25 @@ impl States { pub fn discard_prospective(&mut self) { for i in self.commit .. self.history.len() { self.history[i] = TransactionState::Dropped; + self.previous_transaction[i] = self.commit; } self.history.push(TransactionState::Pending); + self.previous_transaction.push(self.commit); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.commit = self.history.len(); self.history.push(TransactionState::Pending); + for i in 0..self.history.len() - 1 { + self.previous_transaction[i] = self.commit; + } + self.previous_transaction.push(self.commit); } /// Create a new transactional layer. pub fn start_transaction(&mut self) { + self.previous_transaction.push(self.history.len()); self.history.push(TransactionState::TxPending); } @@ -243,6 +272,10 @@ impl States { } } self.history.push(TransactionState::Pending); + self.previous_transaction.truncate(i); + let previous = self.previous_transaction.last() + .cloned().unwrap_or(self.commit); + self.previous_transaction.resize(self.history.len(), previous); } /// Commit a transactional layer. @@ -260,24 +293,21 @@ impl States { } } self.history.push(TransactionState::Pending); + self.previous_transaction.truncate(i); + let previous = self.previous_transaction.last() + .cloned().unwrap_or(self.commit); + self.previous_transaction.resize(self.history.len(), previous); } } +#[inline] /// Get previous index of pending state. /// Used to say if it is possible to drop a committed transaction /// state value. /// Committed index is seen as a transaction state. pub fn find_previous_tx_start(states: &States, from: usize) -> usize { - for i in (states.commit .. from).rev() { - match states.history[i] { - TransactionState::TxPending => { - return i; - }, - _ => (), - } - } - states.commit + states.previous_transaction[from] } From cd7113baa346e962e6567e5b8655d76a6239beb4 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 24 Oct 2019 15:46:58 +0200 Subject: [PATCH 066/129] update runtime versions. --- node/runtime/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/runtime/src/lib.rs b/node/runtime/src/lib.rs index 75e1c9eefd34e..56c3a7a081cd3 100644 --- a/node/runtime/src/lib.rs +++ b/node/runtime/src/lib.rs @@ -84,8 +84,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to equal spec_version. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 181, - impl_version: 181, + spec_version: 182, + impl_version: 182, apis: RUNTIME_API_VERSIONS, }; From 2d91b89950534e3d4c030e49b40f3e7615a925d9 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 30 Oct 2019 19:37:56 +0100 Subject: [PATCH 067/129] fix some documentation nits. clear_terminal_values, renamed to drop_last_values and no longer switch index for previous switch. --- core/state-machine/src/overlayed_changes.rs | 2 +- core/utils/historical-data/src/linear.rs | 74 ++++++++++----------- srml/support/src/storage/mod.rs | 1 + 3 files changed, 38 insertions(+), 39 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index fe3595b06c1fb..38b2818c4c15d 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -116,7 +116,7 @@ fn set_with_extrinsic_inner_overlayed_value( value: Option>, extrinsic_index: u32, ) { - let state = states.len() - 1; + let state = states.num_states() - 1; if let Some(current) = h_value.get_mut(states) { if current.index == state { current.value.value = value; diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs index e6efe0dff82e6..5068d28114866 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -37,7 +37,6 @@ pub enum TransactionState { Dropped, } - /// An entry at a given history height. #[derive(Debug, Clone)] #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] @@ -158,22 +157,10 @@ impl History { #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] pub struct States { history: Vec, - // keep trace of previous transaction. - // associatied logic is that of: - // ``` - // use historical_data::linear::States; - // fn find_previous_tx_start(states: &States, from: usize) -> usize { - // for i in (states.commit .. from).rev() { - // match states.history[i] { - // TransactionState::TxPending => { - // return i; - // }, - // _ => (), - // } - // } - // states.commit - // } - // ``` + // Keep track of the transaction positions. + // This is redundant with `history`, but is + // used to avoid some backwards iteration in + // `find_previous_tx_start` function. previous_transaction: Vec, commit: usize, } @@ -197,13 +184,11 @@ impl States { } /// Current number of inner states. - pub fn len(&self) -> usize { + pub fn num_states(&self) -> usize { self.history.len() } - /// Get index of committed layer, this is - /// additional information needed to manage - /// commit and garbage collect. + /// Get index of committed layer. pub fn committed(&self) -> usize { self.commit } @@ -298,7 +283,6 @@ impl States { .cloned().unwrap_or(self.commit); self.previous_transaction.resize(self.history.len(), previous); } - } #[inline] @@ -313,26 +297,29 @@ pub fn find_previous_tx_start(states: &States, from: usize) -> usize { impl History { /// Set a value, it uses a state history as parameter. + /// /// This method uses `get_mut` and does remove pending /// dropped value. pub fn set(&mut self, states: &States, value: V) { + let last_state_index = states.num_states() - 1; if let Some(v) = self.get_mut(states) { - if v.index == states.history.len() - 1 { + if v.index == last_state_index { *v.value = value; return; } + debug_assert!(v.index < last_state_index, "History expects \ + only new values at the latest state"); } self.push_unchecked(HistoricalValue { value, - index: states.history.len() - 1, + index: last_state_index, }); } - /// Access to latest pending value (non dropped state). + /// Access to the latest pending value (non dropped state). /// When possible please prefer `get_mut` as it can free /// some memory. pub fn get(&self, states: &[TransactionState]) -> Option<&V> { - // index is never 0, let mut index = self.len(); if index == 0 { return None; @@ -417,7 +404,6 @@ impl History { } pub fn into_committed(mut self, states: &[TransactionState], committed: usize) -> Option { - // index is never 0, let mut index = self.len(); if index == 0 { return None; @@ -461,7 +447,7 @@ impl History { match states.history[state_index] { TransactionState::TxPending => { if state_index >= previous_transaction { - previous_switch = Some((index, state_index)); + previous_switch = Some(index); } else { if result.is_none() { result = Some((index, state_index)); @@ -471,7 +457,7 @@ impl History { }, TransactionState::Pending => { if state_index >= previous_transaction { - previous_switch = Some((index, state_index)); + previous_switch = Some(index); } else { if result.is_none() { result = Some((index, state_index)); @@ -484,7 +470,7 @@ impl History { TransactionState::Dropped => (), } } - self.clear_terminal_values(result, previous_switch, 0) + self.drop_last_values(result, previous_switch, 0) } @@ -497,7 +483,7 @@ impl History { &mut self, states: &States, prune_to_commit: bool, - ) -> Option> { + ) -> Option> { let mut index = self.len(); if index == 0 { return None; @@ -518,7 +504,7 @@ impl History { } if state_index >= previous_transaction { - previous_switch = Some((index, state_index)); + previous_switch = Some(index); } else { if result.is_none() { result = Some((index, state_index)); @@ -545,21 +531,33 @@ impl History { } else { 0 }; - self.clear_terminal_values(result, previous_switch, deleted) + self.drop_last_values(result, previous_switch, deleted) } - #[inline] - fn clear_terminal_values( + // Function used to remove all last values for `get_mut` and + // `get_mut_pruning`. + // + // It expects the `result` of the iteration that lookup + // for the latest non dropped value (coupled with its state + // index). That is to remove terminal dropped states. + // + // It also does remove values on committed state that are + // not needed (before another committed value). + // `previous_switch` is the index to the first unneeded value. + // + // An index offset is here in case some content was `deleted` before + // this function call. + fn drop_last_values( &mut self, result: Option<(usize, usize)>, - previous_switch: Option<(usize, usize)>, + previous_switch: Option, deleted: usize, - ) -> Option> { + ) -> Option> { if let Some((index, state_index)) = result { if index + 1 - deleted < self.len() { self.truncate(index + 1 - deleted); } - if let Some((switch_index, state_index)) = previous_switch { + if let Some(switch_index) = previous_switch { if let Some(mut value) = self.pop() { self.truncate(switch_index - deleted); value.index = state_index; diff --git a/srml/support/src/storage/mod.rs b/srml/support/src/storage/mod.rs index da6b2f5cdf685..cd31308acfcb9 100644 --- a/srml/support/src/storage/mod.rs +++ b/srml/support/src/storage/mod.rs @@ -28,6 +28,7 @@ pub mod child; pub mod generator; /// Execute under a transactional layer. +/// /// If the result of execution is an error, /// the transactional layer get reverted; otherwhise /// it is committed. From 61dba1ecd5a37c5a25a34126eac92ee2d77a835b Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 1 Nov 2019 10:39:24 +0100 Subject: [PATCH 068/129] Documentation. --- core/utils/historical-data/src/linear.rs | 163 ++++++++++++++++------- srml/support/src/storage/mod.rs | 15 +-- 2 files changed, 122 insertions(+), 56 deletions(-) diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs index 5068d28114866..410ea9fdb87c9 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -15,12 +15,74 @@ // along with Substrate. If not, see . //! Transactional overlay implementation. +//! +//! # Historical data //! -//! This follows a linear succession of states. -//! This contains multiple unbounded transaction layer -//! and an additional top level 'prospective' layer. -//! It only allows linear history (no branch so -//! inner storage is only an array of element). +//! General principle of historical data applies here. +//! Therefore we got a state and values that keep an history +//! in relation to this state. +//! +//! ## state operation +//! +//! State operation, are only changing the state and do not change +//! stored values. +//! Therefore they cannot modify the existing state values. +//! Thus the state for transaction layer is an historic of all +//! state operation that only append new state. +//! Refering to this history is done through the history index. +//! An operation on a value relates to the state by the index of the state +//! the operation occurs at (field `index` of `HistoricalValue`). +//! +//! It should be possible to mutably change the state under the assumption all +//! values related to the state where pruned or by changing all values related. +//! Generally this is not a good idea because use case here should be a +//! states marginal size (in comparison with values). +//! +//! ## value operation +//! +//! Operation are simple update of value, but to allow query at different historic state, +//! the values (`HistoricalValue`) do keep reference to all possible values. +//! +//! The crate favors light state operation that do not force update +//! of values. +//! Therefore values alignment to updated state is done through +//! manual prune call. +//! Limited updates on values can be done on mutable access. +//! +//! # Transactional layer +//! +//! This module is defining a linear transactional state. +//! +//! ## state operation +//! +//! The state history contains multiple possible states `TransactionStates` and +//! a committed index. +//! +//! There is two pending state, both indicate a valid value, but on also indicate +//! the start of a transactional window. +//! Dropping a transaction just invalidate all state from the start of the last +//! transactional window. +//! Committing a transaction fused it transaction window with the previous transaction +//! window (changing a `TxPending` with a `Pending` state). +//! +//! The committed index is lie a upper transaction state and is considered as the start +//! of a transaction, but cannot be dropped with the same operation. +//! If dropped, all state after this index are dropped. +//! If committed its index is updated and all state prior cannot be dropped. +//! +//! ## value operation +//! +//! Here access to latest state value is a reverse iteration over history of value and +//! matching state, up to the first pending state. +//! +//! On mutable access, terminal dropped state or unneeded state values (values in a same transactional +//! window) are dropped. This allows a clean state up to the latest transactional window at a small cost. +//! +//! # Usage +//! +//! All the primitives for a value expect a reference to the state used to build +//! the value. ** Wrong state usage results in undefined behavior. ** + use rstd::vec::Vec; use rstd::vec; @@ -43,7 +105,7 @@ pub enum TransactionState { pub struct HistoricalValue { /// The stored value. pub value: V, - /// The moment in history when the value got set. + /// The moment in history when the value was set. pub index: usize, } @@ -89,8 +151,7 @@ impl Default for History { // Following implementation are here to isolate // buffer specific functions. impl History { - - fn get_state(&self, index: usize) -> HistoricalValue<&V> { + fn get_unchecked(&self, index: usize) -> HistoricalValue<&V> { self.0[index].as_ref() } @@ -113,7 +174,7 @@ impl History { self.0.truncate(index) } - fn truncate_until(&mut self, index: usize) { + fn remove_until(&mut self, index: usize) { if index > 0 { if self.0.spilled() { let owned = rstd::mem::replace(&mut self.0, Default::default()); @@ -142,10 +203,8 @@ impl History { fn mut_ref(&mut self, index: usize) -> &mut V { &mut self.0[index].value } - } - /// States is both an indexed state to query values with history /// and a committed index that indicates a point in time where /// we cannot drop transaction layer. @@ -158,10 +217,17 @@ impl History { pub struct States { history: Vec, // Keep track of the transaction positions. - // This is redundant with `history`, but is - // used to avoid some backwards iteration in - // `find_previous_tx_start` function. - previous_transaction: Vec, + // This is redundant with `history`. + // This is only use to precalculate the result + // of a backward iteration to find the start + // of a transactional window (function + // `transaction_start_windows`). + // + // At many place we could use this instead + // of the `TxPending` state, but `TxPending` + // state is favored, except for `start_windows` + // case. + start_transaction_window: Vec, commit: usize, } @@ -169,7 +235,7 @@ impl Default for States { fn default() -> Self { States { history: vec![TransactionState::Pending], - previous_transaction: vec![0], + start_transaction_window: vec![0], commit: 0, } } @@ -206,10 +272,10 @@ impl States { #[cfg(any(test, feature = "test-helpers"))] pub fn test_vector( history: Vec, - previous_transaction: Vec, + start_transaction_window: Vec, commit: usize, ) -> Self { - States { history, previous_transaction, commit } + States { history, start_transaction_window, commit } } /// Discard prospective changes to state. @@ -217,10 +283,10 @@ impl States { pub fn discard_prospective(&mut self) { for i in self.commit .. self.history.len() { self.history[i] = TransactionState::Dropped; - self.previous_transaction[i] = self.commit; + self.start_transaction_window[i] = self.commit; } self.history.push(TransactionState::Pending); - self.previous_transaction.push(self.commit); + self.start_transaction_window.push(self.commit); } /// Commit prospective changes to state. @@ -228,14 +294,14 @@ impl States { self.commit = self.history.len(); self.history.push(TransactionState::Pending); for i in 0..self.history.len() - 1 { - self.previous_transaction[i] = self.commit; + self.start_transaction_window[i] = self.commit; } - self.previous_transaction.push(self.commit); + self.start_transaction_window.push(self.commit); } /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.previous_transaction.push(self.history.len()); + self.start_transaction_window.push(self.history.len()); self.history.push(TransactionState::TxPending); } @@ -257,10 +323,10 @@ impl States { } } self.history.push(TransactionState::Pending); - self.previous_transaction.truncate(i); - let previous = self.previous_transaction.last() + self.start_transaction_window.truncate(i); + let previous = self.start_transaction_window.last() .cloned().unwrap_or(self.commit); - self.previous_transaction.resize(self.history.len(), previous); + self.start_transaction_window.resize(self.history.len(), previous); } /// Commit a transactional layer. @@ -278,10 +344,10 @@ impl States { } } self.history.push(TransactionState::Pending); - self.previous_transaction.truncate(i); - let previous = self.previous_transaction.last() + self.start_transaction_window.truncate(i); + let previous = self.start_transaction_window.last() .cloned().unwrap_or(self.commit); - self.previous_transaction.resize(self.history.len(), previous); + self.start_transaction_window.resize(self.history.len(), previous); } } @@ -291,8 +357,8 @@ impl States { /// Used to say if it is possible to drop a committed transaction /// state value. /// Committed index is seen as a transaction state. -pub fn find_previous_tx_start(states: &States, from: usize) -> usize { - states.previous_transaction[from] +pub fn transaction_start_windows(states: &States, from: usize) -> usize { + states.start_transaction_window[from] } impl History { @@ -327,7 +393,7 @@ impl History { debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let HistoricalValue { value, index: state_index } = self.get_state(index); + let HistoricalValue { value, index: state_index } = self.get_unchecked(index); match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending @@ -347,7 +413,7 @@ impl History { debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let state_index = self.get_state(index).index; + let state_index = self.get_unchecked(index).index; match states[state_index] { TransactionState::Dropped => (), TransactionState::Pending @@ -370,7 +436,7 @@ impl History { debug_assert!(states.history.len() >= index); while index > states.commit { index -= 1; - let HistoricalValue { value, index: state_index } = self.get_state(index); + let HistoricalValue { value, index: state_index } = self.get_unchecked(index); match states.history[state_index] { TransactionState::Dropped => (), TransactionState::Pending @@ -390,7 +456,7 @@ impl History { debug_assert!(states.history.len() >= index); while index > 0 { index -= 1; - let HistoricalValue { value, index: state_index } = self.get_state(index); + let HistoricalValue { value, index: state_index } = self.get_unchecked(index); if state_index < states.commit { match states.history[state_index] { TransactionState::Dropped => (), @@ -411,7 +477,7 @@ impl History { debug_assert!(states.len() >= index); while index > 0 { index -= 1; - let state_index = self.get_state(index).index; + let state_index = self.get_unchecked(index).index; if state_index < committed { match states[state_index] { TransactionState::Dropped => (), @@ -428,7 +494,10 @@ impl History { /// Access to latest pending value (non dropped state). /// - /// This method removes latest dropped values up to the latest valid value. + /// This method removes latest dropped and merged values, + /// only keeping the latest valid value. + /// + /// Returns mutable latest pending value. pub fn get_mut( &mut self, states: &States, @@ -439,14 +508,14 @@ impl History { } debug_assert!(states.history.len() >= index); let mut result = None; - let mut previous_transaction = usize::max_value(); + let mut start_transaction_window = usize::max_value(); let mut previous_switch = None; while index > 0 { index -= 1; - let state_index = self.get_state(index).index; + let state_index = self.get_unchecked(index).index; match states.history[state_index] { TransactionState::TxPending => { - if state_index >= previous_transaction { + if state_index >= start_transaction_window { previous_switch = Some(index); } else { if result.is_none() { @@ -456,12 +525,12 @@ impl History { break; }, TransactionState::Pending => { - if state_index >= previous_transaction { + if state_index >= start_transaction_window { previous_switch = Some(index); } else { if result.is_none() { result = Some((index, state_index)); - previous_transaction = find_previous_tx_start(states, state_index); + start_transaction_window = transaction_start_windows(states, state_index); } else { break; } @@ -491,11 +560,11 @@ impl History { let mut prune_index = 0; debug_assert!(states.history.len() >= index); let mut result = None; - let mut previous_transaction = usize::max_value(); + let mut start_transaction_window = usize::max_value(); let mut previous_switch = None; while index > 0 { index -= 1; - let state_index = self.get_state(index).index; + let state_index = self.get_unchecked(index).index; match states.history[state_index] { state @ TransactionState::TxPending | state @ TransactionState::Pending => { @@ -503,13 +572,13 @@ impl History { prune_index = index; } - if state_index >= previous_transaction { + if state_index >= start_transaction_window { previous_switch = Some(index); } else { if result.is_none() { result = Some((index, state_index)); if state == TransactionState::Pending { - previous_transaction = find_previous_tx_start(states, state_index); + start_transaction_window = transaction_start_windows(states, state_index); } } else { if prune_to_commit { @@ -526,7 +595,7 @@ impl History { } } let deleted = if prune_to_commit && prune_index > 0 && result.is_some() { - self.truncate_until(prune_index); + self.remove_until(prune_index); prune_index } else { 0 diff --git a/srml/support/src/storage/mod.rs b/srml/support/src/storage/mod.rs index cd31308acfcb9..e2d15e4a4707e 100644 --- a/srml/support/src/storage/mod.rs +++ b/srml/support/src/storage/mod.rs @@ -34,16 +34,13 @@ pub mod generator; /// it is committed. pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { runtime_io::storage_start_transaction(); - match f() { - Ok(r) => { - runtime_io::storage_commit_transaction(); - Ok(r) - }, - Err(e) => { - runtime_io::storage_discard_transaction(); - Err(e) - } + let r = f(); + if r.is_ok() { + runtime_io::storage_commit_transaction(); + } else { + runtime_io::storage_discard_transaction(); } + r } /// A trait for working with macro-generated storage values under the substrate storage API. From 6334c1088c0a2cb7074e6a045f0818c08c6a93fa Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 1 Nov 2019 11:00:34 +0100 Subject: [PATCH 069/129] favor for expression when possible. --- core/utils/historical-data/src/linear.rs | 63 +++++++++++------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs index 410ea9fdb87c9..523d5d9ccc2a2 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -386,13 +386,12 @@ impl History { /// When possible please prefer `get_mut` as it can free /// some memory. pub fn get(&self, states: &[TransactionState]) -> Option<&V> { - let mut index = self.len(); - if index == 0 { + let self_len = self.len(); + if self_len == 0 { return None; } - debug_assert!(states.len() >= index); - while index > 0 { - index -= 1; + debug_assert!(states.len() >= self_len); + for index in (0 .. self_len).rev() { let HistoricalValue { value, index: state_index } = self.get_unchecked(index); match states[state_index] { TransactionState::Dropped => (), @@ -406,13 +405,12 @@ impl History { /// Get latest value, consuming the historical data. pub fn into_pending(mut self, states: &[TransactionState]) -> Option { - let mut index = self.len(); - if index == 0 { + let self_len = self.len(); + if self_len == 0 { return None; } - debug_assert!(states.len() >= index); - while index > 0 { - index -= 1; + debug_assert!(states.len() >= self_len); + for index in (0 .. self_len).rev() { let state_index = self.get_unchecked(index).index; match states[state_index] { TransactionState::Dropped => (), @@ -429,13 +427,12 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] pub fn get_prospective(&self, states: &States) -> Option<&V> { - let mut index = self.len(); - if index == 0 { + let self_len = self.len(); + if self_len == 0 { return None; } - debug_assert!(states.history.len() >= index); - while index > states.commit { - index -= 1; + debug_assert!(states.history.len() >= self_len); + for index in (states.commit .. self_len).rev() { let HistoricalValue { value, index: state_index } = self.get_unchecked(index); match states.history[state_index] { TransactionState::Dropped => (), @@ -449,13 +446,12 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] pub fn get_committed(&self, states: &States) -> Option<&V> { - let mut index = self.len(); - if index == 0 { + let self_len = self.len(); + if self_len == 0 { return None; } - debug_assert!(states.history.len() >= index); - while index > 0 { - index -= 1; + debug_assert!(states.history.len() >= self_len); + for index in (0 .. self_len).rev() { let HistoricalValue { value, index: state_index } = self.get_unchecked(index); if state_index < states.commit { match states.history[state_index] { @@ -470,13 +466,12 @@ impl History { } pub fn into_committed(mut self, states: &[TransactionState], committed: usize) -> Option { - let mut index = self.len(); - if index == 0 { + let self_len = self.len(); + if self_len == 0 { return None; } - debug_assert!(states.len() >= index); - while index > 0 { - index -= 1; + debug_assert!(states.len() >= self_len); + for index in (0 .. self_len).rev() { let state_index = self.get_unchecked(index).index; if state_index < committed { match states[state_index] { @@ -502,16 +497,15 @@ impl History { &mut self, states: &States, ) -> Option> { - let mut index = self.len(); - if index == 0 { + let self_len = self.len(); + if self_len == 0 { return None; } - debug_assert!(states.history.len() >= index); + debug_assert!(states.history.len() >= self_len); let mut result = None; let mut start_transaction_window = usize::max_value(); let mut previous_switch = None; - while index > 0 { - index -= 1; + for index in (0 .. self_len).rev() { let state_index = self.get_unchecked(index).index; match states.history[state_index] { TransactionState::TxPending => { @@ -553,17 +547,16 @@ impl History { states: &States, prune_to_commit: bool, ) -> Option> { - let mut index = self.len(); - if index == 0 { + let self_len = self.len(); + if self_len == 0 { return None; } let mut prune_index = 0; - debug_assert!(states.history.len() >= index); + debug_assert!(states.history.len() >= self_len); let mut result = None; let mut start_transaction_window = usize::max_value(); let mut previous_switch = None; - while index > 0 { - index -= 1; + for index in (0 .. self_len).rev() { let state_index = self.get_unchecked(index).index; match states.history[state_index] { state @ TransactionState::TxPending From 1b9ff7b301aea8ee7038095bc57028957c549702 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 1 Nov 2019 11:29:43 +0100 Subject: [PATCH 070/129] use 'get_unchecked_pending' for a recurrent pattern matching. --- core/utils/historical-data/src/linear.rs | 58 ++++++++++-------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs index 523d5d9ccc2a2..2df9c6b9b888f 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear.rs @@ -362,6 +362,18 @@ pub fn transaction_start_windows(states: &States, from: usize) -> usize { } impl History { + fn get_pending_unchecked(&self, states: &[TransactionState], history_index: usize) + -> Option> { + let HistoricalValue { value, index } = self.get_unchecked(history_index); + match states[index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending => + return Some(HistoricalValue { value, index }), + } + None + } + /// Set a value, it uses a state history as parameter. /// /// This method uses `get_mut` and does remove pending @@ -392,12 +404,8 @@ impl History { } debug_assert!(states.len() >= self_len); for index in (0 .. self_len).rev() { - let HistoricalValue { value, index: state_index } = self.get_unchecked(index); - match states[state_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending => - return Some(value), + if let Some(h) = self.get_pending_unchecked(states, index) { + return Some(h.value); } } None @@ -411,14 +419,9 @@ impl History { } debug_assert!(states.len() >= self_len); for index in (0 .. self_len).rev() { - let state_index = self.get_unchecked(index).index; - match states[state_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending => { - self.truncate(index + 1); - return self.pop().map(|v| v.value); - }, + if self.get_pending_unchecked(states, index).is_some() { + self.truncate(index + 1); + return self.pop().map(|v| v.value); } } None @@ -433,12 +436,8 @@ impl History { } debug_assert!(states.history.len() >= self_len); for index in (states.commit .. self_len).rev() { - let HistoricalValue { value, index: state_index } = self.get_unchecked(index); - match states.history[state_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending => - return Some(value), + if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { + return Some(h.value); } } None @@ -452,13 +451,9 @@ impl History { } debug_assert!(states.history.len() >= self_len); for index in (0 .. self_len).rev() { - let HistoricalValue { value, index: state_index } = self.get_unchecked(index); - if state_index < states.commit { - match states.history[state_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending => - return Some(value), + if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { + if h.index < states.commit { + return Some(h.value); } } } @@ -472,15 +467,10 @@ impl History { } debug_assert!(states.len() >= self_len); for index in (0 .. self_len).rev() { - let state_index = self.get_unchecked(index).index; - if state_index < committed { - match states[state_index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending => { + if let Some(h) = self.get_pending_unchecked(states, index) { + if h.index < committed { self.truncate(index + 1); return self.pop().map(|v| v.value); - }, } } } From 55cacc1fa816fe05981c2226b74c81fcadf0ce73 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 1 Nov 2019 17:45:40 +0100 Subject: [PATCH 071/129] Reorganize historical crate and documentation. --- core/state-machine/src/changes_trie/build.rs | 3 +- core/state-machine/src/overlayed_changes.rs | 3 +- core/utils/historical-data/src/lib.rs | 62 ++++++- core/utils/historical-data/src/linear/mod.rs | 126 ++++++++++++++ .../src/{linear.rs => linear/transaction.rs} | 155 ++---------------- 5 files changed, 200 insertions(+), 149 deletions(-) create mode 100644 core/utils/historical-data/src/linear/mod.rs rename core/utils/historical-data/src/{linear.rs => linear/transaction.rs} (76%) diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index c8af76ec8a7a9..e57cbf8511889 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -325,7 +325,8 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historical_data::linear::{History, States, HistoricalValue, TransactionState}; + use historical_data::linear::transaction::{States, TransactionState}; + use historical_data::linear::{History, HistoricalValue}; use super::*; fn prepare_for_build(zero: u64) -> ( diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 38b2818c4c15d..ee5b5172e0003 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -22,7 +22,8 @@ use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use historical_data::linear::{States, History, HistoricalValue}; +use historical_data::linear::{History, HistoricalValue}; +use historical_data::linear::transaction::{States}; use historical_data::DEFAULT_GC_CONF; /// The overlayed changes to state to be queried on top of the backend. diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index 2b1259eea4fa8..7ad79cb19fb58 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -14,21 +14,75 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Data storage containing multiple state for a -//! key. -//! Can be use to store historical information -//! on a per item basis. +//! Data storage containing multiple state for a value. +//! Should be use to store historical information for an item. +//! +//! # Design +//! +//! Two elements are used, a global state and multiple historical +//! values. +//! Multiple states for an historical value only make sense if the states +//! got a common basis, that is using partial reference (subsets of a +//! unique global set). +//! +//! ## state operation +//! +//! The state is an historic of state and can be seen as an index +//! for querying or updated the value. +//! State operations, are only changing the state, but not the +//! related stored value (allowing low cost update). +//! If the state cannot modify the existing values, it means that +//! the state cannot change its indexing, it can change a state for +//! an index but cannot remove it, thus it is an append only structure. +//! +//! An operation on a value relates to the state by the last index of the state +//! for the value operation (field `index` of `HistoricalValue`). +//! +//! Therefore the state can be used trimed from its latest index to query a +//! previous state. +//! +//! It should be possible to mutably reduce the state under the assumption +//! that all linked values where pruned accordingly (or by executing an +//! index adjustment on all values atomically). +//! Generally this is not a good idea, as it defeats the purpose of this crate: +//! state size is marginal in comparison of values size. +//! Additionally values query should be very moderatly impacted by the state size +//! (for linear state it is not, but for a tree state the indexing between two branches +//! involves a small additional cost to jump between branch). +//! +//! ## value operation +//! +//! We said before that state operations do not update values, so the value operations +//! will always use the state (or part of the state) as a parameter in order +//! return a different value depending on this state parameter. +//! Operation are simply update of value and queries, but to allow query at different +//! historic state, the values (`HistoricalValue`) do keep reference to all possible values. +//! +//! The crate favors light state operation that do not force update of values, +//! but sometime limited updates on values mutable access can be done (if the cost is +//! close to query cost). +//! To avoid memory exhaustion, a costy manual pruning for value corresponding to dead state +//! is provided (usually call `get_mut_pruning`). +//! It can be use to set in place a garbage collection. + #![cfg_attr(not(feature = "std"), no_std)] pub mod linear; +/// A default sample configuration to manage garbage collection +/// triggering. pub const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { trigger_transaction_gc: 100_000, trigger_commit_gc: 10_000, add_content_size_unit: 64, }; +/// Garbage collection configuration. +/// It is designed to listen on two different operation, transaction +/// and commit. +/// It match transcational semantic with transaction for transaction +/// operation and commit for prospective operation. pub struct GCConfiguration { /// Treshold in number of operation before running a garbage collection. /// diff --git a/core/utils/historical-data/src/linear/mod.rs b/core/utils/historical-data/src/linear/mod.rs new file mode 100644 index 0000000000000..c8e244a03aee8 --- /dev/null +++ b/core/utils/historical-data/src/linear/mod.rs @@ -0,0 +1,126 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Linear historical state container. + + +pub mod transaction; + +/// An entry at a given history height. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct HistoricalValue { + /// The stored value. + pub value: V, + /// The moment in history when the value was set. + pub index: usize, +} + +impl From<(V, usize)> for HistoricalValue { + fn from(input: (V, usize)) -> HistoricalValue { + HistoricalValue { value: input.0, index: input.1 } + } +} + +impl HistoricalValue { + fn as_ref(&self) -> HistoricalValue<&V> { + HistoricalValue { + value: &self.value, + index: self.index, + } + } +} + +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. +type MemoryOnly = smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>; + +/// Size of preallocated history per element. +/// Currently at two for committed and prospective only. +/// It means that using transaction in a module got a direct allocation cost. +const ALLOCATED_HISTORY: usize = 2; + +/// History of value that are related to a state history (eg field `history` of +/// an `OverlayedChangeSet`). +/// +/// Values are always paired with a state history index. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct History(MemoryOnly); + +impl Default for History { + fn default() -> Self { + History(Default::default()) + } +} + +// Following implementation are here to isolate +// buffer specific functions. +impl History { + fn get_unchecked(&self, index: usize) -> HistoricalValue<&V> { + self.0[index].as_ref() + } + + #[cfg(any(test, feature = "test-helpers"))] + /// Create an history from an existing history. + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = History::default(); + for v in input { + history.push_unchecked(v); + } + history + } + + /// Current number of inner states. + pub fn len(&self) -> usize { + self.0.len() + } + + fn truncate(&mut self, index: usize) { + self.0.truncate(index) + } + + fn remove_until(&mut self, index: usize) { + if index > 0 { + if self.0.spilled() { + let owned = rstd::mem::replace(&mut self.0, Default::default()); + self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); + } else { + for i in (0..index).rev() { + self.0.remove(i); + } + } + } + } + + fn pop(&mut self) -> Option> { + self.0.pop() + } + + /// Append without checking if a value already exist. + /// If a value already exists, the history will be broken. + /// This method shall only be call after a `get_mut` where + /// the returned index indicate that a `set` will result + /// in appending a value. + pub fn push_unchecked(&mut self, value: HistoricalValue) { + self.0.push(value) + } + + fn mut_ref(&mut self, index: usize) -> &mut V { + &mut self.0[index].value + } +} diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear/transaction.rs similarity index 76% rename from core/utils/historical-data/src/linear.rs rename to core/utils/historical-data/src/linear/transaction.rs index 2df9c6b9b888f..8d02207949478 100644 --- a/core/utils/historical-data/src/linear.rs +++ b/core/utils/historical-data/src/linear/transaction.rs @@ -14,42 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Transactional overlay implementation. +//! Transactional layer implementation. //! -//! # Historical data -//! -//! General principle of historical data applies here. -//! Therefore we got a state and values that keep an history -//! in relation to this state. -//! -//! ## state operation -//! -//! State operation, are only changing the state and do not change -//! stored values. -//! Therefore they cannot modify the existing state values. -//! Thus the state for transaction layer is an historic of all -//! state operation that only append new state. -//! Refering to this history is done through the history index. -//! An operation on a value relates to the state by the index of the state -//! the operation occurs at (field `index` of `HistoricalValue`). -//! -//! It should be possible to mutably change the state under the assumption all -//! values related to the state where pruned or by changing all values related. -//! Generally this is not a good idea because use case here should be a -//! states marginal size (in comparison with values). -//! -//! ## value operation -//! -//! Operation are simple update of value, but to allow query at different historic state, -//! the values (`HistoricalValue`) do keep reference to all possible values. -//! -//! The crate favors light state operation that do not force update -//! of values. -//! Therefore values alignment to updated state is done through -//! manual prune call. -//! Limited updates on values can be done on mutable access. -//! -//! # Transactional layer +//! General principle of historical data (see module design)applies here. +//! Therefore we got a state and values with an history dependant on +//! a state. //! //! This module is defining a linear transactional state. //! @@ -75,9 +44,14 @@ //! Here access to latest state value is a reverse iteration over history of value and //! matching state, up to the first pending state. //! -//! On mutable access, terminal dropped state or unneeded state values (values in a same transactional -//! window) are dropped. This allows a clean state up to the latest transactional window at a small cost. +//! On mutable access, terminal dropped state or unneeded state values (values in a same +//! transactional window) are dropped. +//! This allows a clean state up to the latest transactional window at a small cost. //! +//! ** Update only occurs on a latest state **, therefore state primitives should not allow +//! access to some subset of the state (full state is used as parameter when for query a simple +//! slice is used). +//! //! # Usage //! //! All the primitives for a value expect a reference to the state used to build @@ -86,6 +60,7 @@ use rstd::vec::Vec; use rstd::vec; +use crate::linear::{HistoricalValue, History}; #[derive(Debug, Clone, Eq, PartialEq, Copy)] /// State of a transactional layer. @@ -99,112 +74,6 @@ pub enum TransactionState { Dropped, } -/// An entry at a given history height. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct HistoricalValue { - /// The stored value. - pub value: V, - /// The moment in history when the value was set. - pub index: usize, -} - -impl From<(V, usize)> for HistoricalValue { - fn from(input: (V, usize)) -> HistoricalValue { - HistoricalValue { value: input.0, index: input.1 } - } -} - -impl HistoricalValue { - fn as_ref(&self) -> HistoricalValue<&V> { - HistoricalValue { - value: &self.value, - index: self.index, - } - } -} - -/// Array like buffer for in memory storage. -/// By in memory we expect that this will -/// not required persistence and is not serialized. -type MemoryOnly = smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>; - -/// Size of preallocated history per element. -/// Currently at two for committed and prospective only. -/// It means that using transaction in a module got a direct allocation cost. -const ALLOCATED_HISTORY: usize = 2; - -/// History of value that are related to a state history (eg field `history` of -/// an `OverlayedChangeSet`). -/// -/// Values are always paired with a state history index. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct History(MemoryOnly); - -impl Default for History { - fn default() -> Self { - History(Default::default()) - } -} - -// Following implementation are here to isolate -// buffer specific functions. -impl History { - fn get_unchecked(&self, index: usize) -> HistoricalValue<&V> { - self.0[index].as_ref() - } - - #[cfg(any(test, feature = "test-helpers"))] - /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator>) -> Self { - let mut history = History::default(); - for v in input { - history.push_unchecked(v); - } - history - } - - /// Current number of inner states. - pub fn len(&self) -> usize { - self.0.len() - } - - fn truncate(&mut self, index: usize) { - self.0.truncate(index) - } - - fn remove_until(&mut self, index: usize) { - if index > 0 { - if self.0.spilled() { - let owned = rstd::mem::replace(&mut self.0, Default::default()); - self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); - } else { - for i in (0..index).rev() { - self.0.remove(i); - } - } - } - } - - fn pop(&mut self) -> Option> { - self.0.pop() - } - - /// Append without checking if a value already exist. - /// If a value already exists, the history will be broken. - /// This method shall only be call after a `get_mut` where - /// the returned index indicate that a `set` will result - /// in appending a value. - pub fn push_unchecked(&mut self, value: HistoricalValue) { - self.0.push(value) - } - - fn mut_ref(&mut self, index: usize) -> &mut V { - &mut self.0[index].value - } -} - /// States is both an indexed state to query values with history /// and a committed index that indicates a point in time where /// we cannot drop transaction layer. From 4920da423e11fe9b296711cfac75482b26e3c8be Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 1 Nov 2019 17:51:12 +0100 Subject: [PATCH 072/129] fix import. --- core/test-runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 5b8c915e82cfe..2392109abdc7d 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -916,7 +916,7 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { } fn test_historical_data() { - let mut states = historical_data::linear::States::default(); + let mut states = historical_data::linear::transaction::States::default(); let mut value = historical_data::linear::History::default(); if value.get(states.as_ref()) != None { panic!("Got a value for empty data"); From 4a1fb7f194fd9e9c4bde7905297f90a9e4c57502 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 11 Nov 2019 13:42:39 +0100 Subject: [PATCH 073/129] Preparing refactor. --- core/state-machine/src/changes_trie/build.rs | 2 +- core/state-machine/src/overlayed_changes.rs | 4 +- core/test-runtime/src/lib.rs | 2 +- .../src/detached/linear/mod.rs | 19 +++ .../src/{ => detached}/linear/transaction.rs | 0 core/utils/historical-data/src/lib.rs | 102 +------------- core/utils/historical-data/src/linear/mod.rs | 126 ------------------ 7 files changed, 27 insertions(+), 228 deletions(-) create mode 100644 core/utils/historical-data/src/detached/linear/mod.rs rename core/utils/historical-data/src/{ => detached}/linear/transaction.rs (100%) delete mode 100644 core/utils/historical-data/src/linear/mod.rs diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index e57cbf8511889..150ef66cb51e3 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -325,7 +325,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historical_data::linear::transaction::{States, TransactionState}; + use historical_data::detached::linear::transaction::{States, TransactionState}; use historical_data::linear::{History, HistoricalValue}; use super::*; diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index ee5b5172e0003..6fe7f82d8834c 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -23,8 +23,8 @@ use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; use historical_data::linear::{History, HistoricalValue}; -use historical_data::linear::transaction::{States}; -use historical_data::DEFAULT_GC_CONF; +use historical_data::detached::linear::transaction::{States}; +use historical_data::detached::DEFAULT_GC_CONF; /// The overlayed changes to state to be queried on top of the backend. /// diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 2392109abdc7d..3a5b548adaa5c 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -916,7 +916,7 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { } fn test_historical_data() { - let mut states = historical_data::linear::transaction::States::default(); + let mut states = historical_data::detached::linear::transaction::States::default(); let mut value = historical_data::linear::History::default(); if value.get(states.as_ref()) != None { panic!("Got a value for empty data"); diff --git a/core/utils/historical-data/src/detached/linear/mod.rs b/core/utils/historical-data/src/detached/linear/mod.rs new file mode 100644 index 0000000000000..736360ce584a9 --- /dev/null +++ b/core/utils/historical-data/src/detached/linear/mod.rs @@ -0,0 +1,19 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Linear detached historical. + +pub mod transaction; diff --git a/core/utils/historical-data/src/linear/transaction.rs b/core/utils/historical-data/src/detached/linear/transaction.rs similarity index 100% rename from core/utils/historical-data/src/linear/transaction.rs rename to core/utils/historical-data/src/detached/linear/transaction.rs diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index 7ad79cb19fb58..f39baab7a1a40 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -15,106 +15,12 @@ // along with Substrate. If not, see . //! Data storage containing multiple state for a value. -//! Should be use to store historical information for an item. -//! -//! # Design -//! -//! Two elements are used, a global state and multiple historical -//! values. -//! Multiple states for an historical value only make sense if the states -//! got a common basis, that is using partial reference (subsets of a -//! unique global set). -//! -//! ## state operation -//! -//! The state is an historic of state and can be seen as an index -//! for querying or updated the value. -//! State operations, are only changing the state, but not the -//! related stored value (allowing low cost update). -//! If the state cannot modify the existing values, it means that -//! the state cannot change its indexing, it can change a state for -//! an index but cannot remove it, thus it is an append only structure. -//! -//! An operation on a value relates to the state by the last index of the state -//! for the value operation (field `index` of `HistoricalValue`). -//! -//! Therefore the state can be used trimed from its latest index to query a -//! previous state. -//! -//! It should be possible to mutably reduce the state under the assumption -//! that all linked values where pruned accordingly (or by executing an -//! index adjustment on all values atomically). -//! Generally this is not a good idea, as it defeats the purpose of this crate: -//! state size is marginal in comparison of values size. -//! Additionally values query should be very moderatly impacted by the state size -//! (for linear state it is not, but for a tree state the indexing between two branches -//! involves a small additional cost to jump between branch). -//! -//! ## value operation -//! -//! We said before that state operations do not update values, so the value operations -//! will always use the state (or part of the state) as a parameter in order -//! return a different value depending on this state parameter. -//! Operation are simply update of value and queries, but to allow query at different -//! historic state, the values (`HistoricalValue`) do keep reference to all possible values. -//! -//! The crate favors light state operation that do not force update of values, -//! but sometime limited updates on values mutable access can be done (if the cost is -//! close to query cost). -//! To avoid memory exhaustion, a costy manual pruning for value corresponding to dead state -//! is provided (usually call `get_mut_pruning`). -//! It can be use to set in place a garbage collection. +//! This is use to store historical information for an item, +//! and does have to include any kind of proof. #![cfg_attr(not(feature = "std"), no_std)] +pub mod synch; +pub mod detached; pub mod linear; - -/// A default sample configuration to manage garbage collection -/// triggering. -pub const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { - trigger_transaction_gc: 100_000, - trigger_commit_gc: 10_000, - add_content_size_unit: 64, -}; - -/// Garbage collection configuration. -/// It is designed to listen on two different operation, transaction -/// and commit. -/// It match transcational semantic with transaction for transaction -/// operation and commit for prospective operation. -pub struct GCConfiguration { - /// Treshold in number of operation before running a garbage collection. - /// - /// Should be same as `TRIGGER_COMMIT_GC` or higher - /// (we most likely do not want lower as transaction are - /// possibly more frequent than commit). - pub trigger_transaction_gc: usize, - - /// Treshold in number of operation before running a garbage colletion - /// on a commit operation. - /// - /// We may want a lower value than for a transaction, even - /// a 1 if we want to do it between every operation. - pub trigger_commit_gc: usize, - - /// Used to count big content as multiple operations. - /// This is a number of octet. - /// Set to 0 to ignore. - pub add_content_size_unit: usize, -} - -impl GCConfiguration { - /// Cost depending on value if any. - pub fn operation_cost(&self, val: Option<&rstd::vec::Vec>) -> usize { - let additional_cost = if self.add_content_size_unit > 0 { - if let Some(s) = val.as_ref() { - s.len() / self.add_content_size_unit - } else { - 0 - } - } else { 0 }; - 1 + additional_cost - } - -} diff --git a/core/utils/historical-data/src/linear/mod.rs b/core/utils/historical-data/src/linear/mod.rs deleted file mode 100644 index c8e244a03aee8..0000000000000 --- a/core/utils/historical-data/src/linear/mod.rs +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Linear historical state container. - - -pub mod transaction; - -/// An entry at a given history height. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct HistoricalValue { - /// The stored value. - pub value: V, - /// The moment in history when the value was set. - pub index: usize, -} - -impl From<(V, usize)> for HistoricalValue { - fn from(input: (V, usize)) -> HistoricalValue { - HistoricalValue { value: input.0, index: input.1 } - } -} - -impl HistoricalValue { - fn as_ref(&self) -> HistoricalValue<&V> { - HistoricalValue { - value: &self.value, - index: self.index, - } - } -} - -/// Array like buffer for in memory storage. -/// By in memory we expect that this will -/// not required persistence and is not serialized. -type MemoryOnly = smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>; - -/// Size of preallocated history per element. -/// Currently at two for committed and prospective only. -/// It means that using transaction in a module got a direct allocation cost. -const ALLOCATED_HISTORY: usize = 2; - -/// History of value that are related to a state history (eg field `history` of -/// an `OverlayedChangeSet`). -/// -/// Values are always paired with a state history index. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct History(MemoryOnly); - -impl Default for History { - fn default() -> Self { - History(Default::default()) - } -} - -// Following implementation are here to isolate -// buffer specific functions. -impl History { - fn get_unchecked(&self, index: usize) -> HistoricalValue<&V> { - self.0[index].as_ref() - } - - #[cfg(any(test, feature = "test-helpers"))] - /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator>) -> Self { - let mut history = History::default(); - for v in input { - history.push_unchecked(v); - } - history - } - - /// Current number of inner states. - pub fn len(&self) -> usize { - self.0.len() - } - - fn truncate(&mut self, index: usize) { - self.0.truncate(index) - } - - fn remove_until(&mut self, index: usize) { - if index > 0 { - if self.0.spilled() { - let owned = rstd::mem::replace(&mut self.0, Default::default()); - self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); - } else { - for i in (0..index).rev() { - self.0.remove(i); - } - } - } - } - - fn pop(&mut self) -> Option> { - self.0.pop() - } - - /// Append without checking if a value already exist. - /// If a value already exists, the history will be broken. - /// This method shall only be call after a `get_mut` where - /// the returned index indicate that a `set` will result - /// in appending a value. - pub fn push_unchecked(&mut self, value: HistoricalValue) { - self.0.push(value) - } - - fn mut_ref(&mut self, index: usize) -> &mut V { - &mut self.0[index].value - } -} From a995e18cbebf7646d3cae7bdaa53cc3c09345d6b Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Tue, 12 Nov 2019 18:11:10 +0100 Subject: [PATCH 074/129] setting up linear synch historical data. --- .../utils/historical-data/src/detached/mod.rs | 118 +++++++++ core/utils/historical-data/src/lib.rs | 35 +++ core/utils/historical-data/src/linear.rs | 102 ++++++++ .../historical-data/src/synch/linear/mod.rs | 19 ++ .../src/synch/linear/transaction.rs | 242 ++++++++++++++++++ core/utils/historical-data/src/synch/mod.rs | 21 ++ 6 files changed, 537 insertions(+) create mode 100644 core/utils/historical-data/src/detached/mod.rs create mode 100644 core/utils/historical-data/src/linear.rs create mode 100644 core/utils/historical-data/src/synch/linear/mod.rs create mode 100644 core/utils/historical-data/src/synch/linear/transaction.rs create mode 100644 core/utils/historical-data/src/synch/mod.rs diff --git a/core/utils/historical-data/src/detached/mod.rs b/core/utils/historical-data/src/detached/mod.rs new file mode 100644 index 0000000000000..b5e701f20b58d --- /dev/null +++ b/core/utils/historical-data/src/detached/mod.rs @@ -0,0 +1,118 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! # Design +//! +//! The historical storage is loosely related to a global detached +//! state. +//! +//! Two elements are used, a global state and multiple historical +//! values. +//! Multiple states for an historical value only make sense if the states +//! got a common basis, that is using partial reference (subsets of a +//! unique global set). +//! +//! ## state operation +//! +//! The state is an historic of state and can be seen as an index +//! for querying or updated the value. +//! State operations, are only changing the state, but not the +//! related stored value (allowing low cost update). +//! If the state cannot modify the existing values, it means that +//! the state cannot change its indexing, it can change a state for +//! an index but cannot remove it, thus it is an append only structure. +//! +//! An operation on a value relates to the state by the last index of the state +//! for the value operation (field `index` of `HistoricalValue`). +//! +//! Therefore the state can be used trimed from its latest index to query a +//! previous state. +//! +//! It should be possible to mutably reduce the state under the assumption +//! that all linked values where pruned accordingly (or by executing an +//! index adjustment on all values atomically). +//! Generally this is not a good idea, as it defeats the purpose of this crate: +//! state size is marginal in comparison of values size. +//! Additionally values query should be very moderatly impacted by the state size +//! (for linear state it is not, but for a tree state the indexing between two branches +//! involves a small additional cost to jump between branch). +//! +//! ## value operation +//! +//! We said before that state operations do not update values, so the value operations +//! will always use the state (or part of the state) as a parameter in order +//! return a different value depending on this state parameter. +//! Operation are simply update of value and queries, but to allow query at different +//! historic state, the values (`HistoricalValue`) do keep reference to all possible values. +//! +//! The crate favors light state operation that do not force update of values, +//! but sometime limited updates on values mutable access can be done (if the cost is +//! close to query cost). +//! To avoid memory exhaustion, a costy manual pruning for value corresponding to dead state +//! is provided (usually call `get_mut_pruning`). +//! It can be use to set in place a garbage collection. + + +pub mod linear; + +/// A default sample configuration to manage garbage collection +/// triggering. +pub const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { + trigger_transaction_gc: 100_000, + trigger_commit_gc: 10_000, + add_content_size_unit: 64, +}; + +/// Garbage collection configuration. +/// It is designed to listen on two different operation, transaction +/// and commit. +/// It match transcational semantic with transaction for transaction +/// operation and commit for prospective operation. +pub struct GCConfiguration { + /// Treshold in number of operation before running a garbage collection. + /// + /// Should be same as `TRIGGER_COMMIT_GC` or higher + /// (we most likely do not want lower as transaction are + /// possibly more frequent than commit). + pub trigger_transaction_gc: usize, + + /// Treshold in number of operation before running a garbage colletion + /// on a commit operation. + /// + /// We may want a lower value than for a transaction, even + /// a 1 if we want to do it between every operation. + pub trigger_commit_gc: usize, + + /// Used to count big content as multiple operations. + /// This is a number of octet. + /// Set to 0 to ignore. + pub add_content_size_unit: usize, +} + +impl GCConfiguration { + /// Cost depending on value if any. + pub fn operation_cost(&self, val: Option<&rstd::vec::Vec>) -> usize { + let additional_cost = if self.add_content_size_unit > 0 { + if let Some(s) = val.as_ref() { + s.len() / self.add_content_size_unit + } else { + 0 + } + } else { 0 }; + 1 + additional_cost + } + +} diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index f39baab7a1a40..bc975439cb207 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -24,3 +24,38 @@ pub mod synch; pub mod detached; pub mod linear; + +/// An entry at a given history index. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +pub struct HistoricalValue { + /// The stored value. + pub value: V, + /// The moment in history when the value got set. + pub index: I, +} + +impl From<(V, I)> for HistoricalValue { + fn from(input: (V, I)) -> HistoricalValue { + HistoricalValue { value: input.0, index: input.1 } + } +} + +impl HistoricalValue { + fn as_ref(&self) -> HistoricalValue<&V, I> { + HistoricalValue { value: &self.value, index: self.index.clone() } + } + + fn as_mut(&mut self) -> HistoricalValue<&mut V, I> { + HistoricalValue { value: &mut self.value, index: self.index.clone() } + } +} + +#[cfg_attr(any(test, feature = "test"), derive(PartialEq, Debug))] +/// Prunning result to be able to proceed +/// with further update if the value needs it. +pub enum PruneResult { + Unchanged, + Changed, + Cleared, +} diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs new file mode 100644 index 0000000000000..5fdd090cc6aba --- /dev/null +++ b/core/utils/historical-data/src/linear.rs @@ -0,0 +1,102 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Linear historical state container. + + +/// An entry at a given history height. +pub type HistoricalValue = crate::HistoricalValue; + +/// Array like buffer for in memory storage. +/// By in memory we expect that this will +/// not required persistence and is not serialized. +type MemoryOnly = smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>; + +/// Size of preallocated history per element. +/// Currently at two for committed and prospective only. +/// It means that using transaction in a module got a direct allocation cost. +const ALLOCATED_HISTORY: usize = 2; + +/// History of value that are related to a state history (eg field `history` of +/// an `OverlayedChangeSet`). +/// +/// Values are always paired with a state history index. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct History(pub(crate) MemoryOnly); + +impl Default for History { + fn default() -> Self { + History(Default::default()) + } +} + +// Following implementation are here to isolate +// buffer specific functions. +impl History { + pub(crate) fn get_unchecked(&self, index: usize) -> HistoricalValue<&V> { + self.0[index].as_ref() + } + + #[cfg(any(test, feature = "test-helpers"))] + /// Create an history from an existing history. + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = History::default(); + for v in input { + history.push_unchecked(v); + } + history + } + + /// Current number of inner states. + pub fn len(&self) -> usize { + self.0.len() + } + + pub(crate) fn truncate(&mut self, index: usize) { + self.0.truncate(index) + } + + pub(crate) fn remove_until(&mut self, index: usize) { + if index > 0 { + if self.0.spilled() { + let owned = rstd::mem::replace(&mut self.0, Default::default()); + self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); + } else { + for i in (0..index).rev() { + self.0.remove(i); + } + } + } + } + + pub(crate) fn pop(&mut self) -> Option> { + self.0.pop() + } + + /// Append without checking if a value already exist. + /// If a value already exists, the history will be broken. + /// This method shall only be call after a `get_mut` where + /// the returned index indicate that a `set` will result + /// in appending a value. + pub fn push_unchecked(&mut self, value: HistoricalValue) { + self.0.push(value) + } + + pub(crate) fn mut_ref(&mut self, index: usize) -> &mut V { + &mut self.0[index].value + } +} diff --git a/core/utils/historical-data/src/synch/linear/mod.rs b/core/utils/historical-data/src/synch/linear/mod.rs new file mode 100644 index 0000000000000..e9a35ef64b51b --- /dev/null +++ b/core/utils/historical-data/src/synch/linear/mod.rs @@ -0,0 +1,19 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Linear arrangement of historical data. + +pub mod transaction; diff --git a/core/utils/historical-data/src/synch/linear/transaction.rs b/core/utils/historical-data/src/synch/linear/transaction.rs new file mode 100644 index 0000000000000..b73f5441582a6 --- /dev/null +++ b/core/utils/historical-data/src/synch/linear/transaction.rs @@ -0,0 +1,242 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Linear arrangement of historical data with transactional +//! support. +//! +//! +//! # Global state +//! +//! The only global state is a counter of overlayed transaction layer. +//! Committing or discarding a layer must use this counter. +//! +//! # Local state +//! +//! Local state is either a committed state (this is a single first independant level +//! of transaction) or a reference to the transaction counter in use in time of creation. + +use rstd::vec::Vec; +use crate::PruneResult; + +/// Global state is a simple counter to the current overlay layer index. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct States(usize); + +impl Default for States { + fn default() -> Self { + States(0) + } +} + +impl States { + + /// Build any state for testing only. + #[cfg(any(test, feature = "test-helpers"))] + pub fn test_state( + current_layer_number: usize, + ) -> Self { + States(current_layer_number) + } + + /// Discard prospective changes to state. + /// It does not reverts actual values. + /// A subsequent synchronisation of stored values is needed. + pub fn discard_prospective(&mut self) { + if self.0 > 0 { + self.0 -= 1; + } + } + + /// Update a value to a new prospective. + pub fn apply_discard_prospective(&self) { + unimplemented!("TODO History as mut param"); + } + + /// Commit prospective changes to state. + /// A subsequent synchronisation of stored values is needed. + pub fn commit_prospective(&mut self) { + if self.0 > 0 { + self.0 -= 1; + } + } + + /// Update a value to a new prospective. + /// Multiple commit can be applied at the same time. + pub fn apply_commit_prospective(&self) { + unimplemented!("TODO History as mut param"); + } + + + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.0 += 1; + } + + /// Discard a transactional layer. + /// It does not reverts actual values. + /// A subsequent synchronisation of stored values is needed. + pub fn discard_transaction(&mut self) { + if self.0 > 0 { + self.0 -= 1; + } + } + + /// Update a value to previous transaction. + /// Multiple discard can be applied at the same time. + /// Returns true if value is still needed. + pub fn apply_discard_transaction(&self) -> PruneResult { + unimplemented!("TODO History as mut param"); + } + + /// Discard a transactional layer. + /// It does not reverts actual values. + /// A subsequent synchronisation of stored values is needed. + pub fn commit_transaction(&mut self) { + if self.0 > 0 { + self.0 -= 1; + } + } + + /// Update a value to be the best historical value + /// after one or more `commit_transaction` calls. + /// Multiple discard can be applied at the same time. + /// Returns true if value is still needed. + pub fn apply_commit_transaction(&self) -> PruneResult { + unimplemented!("TODO History as mut param"); + } + +} + +/// Possible state for a historical value, committed +/// is not touched by transactional action, transaction +/// stored the transaction index of insertion. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub enum State { + Committed, + Transaction(usize), +} +impl State { + fn transaction_index(&self) -> Option { + if let &State::Transaction(ix) = self { + Some(ix) + } else { + None + } + } +} +/// An entry at a given history height. +pub type HistoricalValue = crate::HistoricalValue; + +/// History of value and their state. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct History(pub(crate) Vec>); + +impl Default for History { + fn default() -> Self { + History(Default::default()) + } +} + +impl History { + /// Set a value, it uses a global state as parameter. + pub fn set(&mut self, states: &States, value: V) { + if let Some(v) = self.0.last_mut() { + debug_assert!(v.index.transaction_index().unwrap_or(0) <= states.0, + "History expects \ + only new values at the latest state, some state has not \ + synchronized properly"); + if v.index.transaction_index() == Some(states.0) { + v.value = value; + return; + } + } + self.0.push(HistoricalValue { + value, + index: State::Transaction(states.0), + }); + } + + /// Access to the latest pending value. + pub fn get(&self) -> Option<&V> { + self.0.last().map(|h| &h.value) + } + + /// Get latest value, consuming the historical data. + pub fn into_pending(mut self) -> Option { + if let Some(v) = self.0.pop() { + Some(v.value) + } else { + None + } + } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn get_prospective(&self) -> Option<&V> { + match self.0.get(0) { + Some(HistoricalValue { + value: _, + index: State::Committed, + }) => { + if let Some(HistoricalValue { + value, + index: State::Transaction(_), + }) = self.0.get(1) { + Some(&value) + } else { + None + } + }, + Some(HistoricalValue { + value, + index: State::Transaction(_), + }) => Some(&value), + None => None, + } + } + + #[cfg(any(test, feature = "test-helpers"))] + pub fn get_committed(&self) -> Option<&V> { + if let Some(HistoricalValue { + value, + index: State::Committed, + }) = self.0.get(0) { + return Some(&value) + } else { + None + } + } + + pub fn into_committed(mut self) -> Option { + self.0.truncate(1); + if let Some(HistoricalValue { + value, + index: State::Committed, + }) = self.0.pop() { + return Some(value) + } else { + None + } + } + + /// Returns mutable latest pending historical value. + pub fn get_mut(&mut self) -> Option> { + self.0.last_mut().map(|h| h.as_mut()) + } + +} diff --git a/core/utils/historical-data/src/synch/mod.rs b/core/utils/historical-data/src/synch/mod.rs new file mode 100644 index 0000000000000..f9e0425b44898 --- /dev/null +++ b/core/utils/historical-data/src/synch/mod.rs @@ -0,0 +1,21 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Synch module contains historical data that do not require +//! a separated state for most operation. Global state operation +//! must therefore apply on all values with history synchronously. + +pub mod linear; From fb018569dd0f3017a3f762aeaba052d3f9a5cc2e Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Tue, 12 Nov 2019 18:42:32 +0100 Subject: [PATCH 075/129] quick implementation before fuzzing. --- .../src/synch/linear/transaction.rs | 94 ++++++++++++++++--- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/core/utils/historical-data/src/synch/linear/transaction.rs b/core/utils/historical-data/src/synch/linear/transaction.rs index b73f5441582a6..1621b30d47b9f 100644 --- a/core/utils/historical-data/src/synch/linear/transaction.rs +++ b/core/utils/historical-data/src/synch/linear/transaction.rs @@ -62,8 +62,19 @@ impl States { } /// Update a value to a new prospective. - pub fn apply_discard_prospective(&self) { - unimplemented!("TODO History as mut param"); + pub fn apply_discard_prospective(&self, value: &mut History) -> PruneResult { + if value.0.len() == 0 { + return PruneResult::Cleared; + } + if value.0[0].index == State::Committed { + if value.0.len() == 1 { + return PruneResult::Unchanged; + } + value.0.truncate(1); + } else { + value.0.clear(); + } + PruneResult::Changed } /// Commit prospective changes to state. @@ -76,11 +87,24 @@ impl States { /// Update a value to a new prospective. /// Multiple commit can be applied at the same time. - pub fn apply_commit_prospective(&self) { - unimplemented!("TODO History as mut param"); + pub fn apply_commit_prospective(&self, value: &mut History) -> PruneResult { + if value.0.len() == 0 { + return PruneResult::Cleared; + } + if value.0.len() == 1 { + if value.0[0].index != State::Committed { + value.0[0].index = State::Committed; + } else { + return PruneResult::Unchanged; + } + } else if let Some(mut v) = value.0.pop() { + v.index = State::Committed; + value.0.clear(); + value.0.push(v); + } + PruneResult::Changed } - /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.0 += 1; @@ -98,8 +122,25 @@ impl States { /// Update a value to previous transaction. /// Multiple discard can be applied at the same time. /// Returns true if value is still needed. - pub fn apply_discard_transaction(&self) -> PruneResult { - unimplemented!("TODO History as mut param"); + pub fn apply_discard_transaction(&self, value: &mut History) -> PruneResult { + let init_len = value.0.len(); + for i in (0 .. value.0.len()).rev() { + if let HistoricalValue { + value: _, + index: State::Transaction(ix), + } = value.0[i] { + if ix > self.0 { + let _ = value.0.pop(); + } else { break } + } else { break } + } + if value.0.len() == 0 { + PruneResult::Cleared + } else if value.0.len() != init_len { + PruneResult::Changed + } else { + PruneResult::Unchanged + } } /// Discard a transactional layer. @@ -115,17 +156,46 @@ impl States { /// after one or more `commit_transaction` calls. /// Multiple discard can be applied at the same time. /// Returns true if value is still needed. - pub fn apply_commit_transaction(&self) -> PruneResult { - unimplemented!("TODO History as mut param"); + pub fn apply_commit_transaction(&self, value: &mut History) -> PruneResult { + let init_len = value.0.len(); + let mut new_value = None; + for i in (0 .. value.0.len()).rev() { + if let HistoricalValue { + value: _, + index: State::Transaction(ix), + } = value.0[i] { + if ix > self.0 { + if let Some(v) = value.0.pop() { + if new_value.is_none() { + new_value = Some(v.value); + } + } + } else if ix == self.0 && new_value.is_some() { + let _ = value.0.pop(); + } else { break } + } else { break } + } + let result = if value.0.len() == 0 { + PruneResult::Cleared + } else if value.0.len() != init_len { + PruneResult::Changed + } else { + PruneResult::Unchanged + }; + if let Some(new_value) = new_value { + value.0.push(HistoricalValue { + value: new_value, + index: State::Transaction(self.0), + }); + } + result } - } /// Possible state for a historical value, committed /// is not touched by transactional action, transaction /// stored the transaction index of insertion. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum State { Committed, Transaction(usize), From ef0bcdaed89e057bf0c6451c47b2c110454c6c11 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 13 Nov 2019 18:03:51 +0100 Subject: [PATCH 076/129] code fix and overlay implemetation, before deletion of old code. --- core/state-machine/fuzz/Cargo.toml | 6 +- ... => fuzz_transactions_with_compactness.rs} | 2 +- core/state-machine/fuzz/src/lib.rs | 11 +- core/state-machine/src/changes_trie/build.rs | 29 ++-- core/state-machine/src/ext.rs | 9 +- core/state-machine/src/lib.rs | 7 +- core/state-machine/src/overlayed_changes.rs | 132 +++++++----------- core/utils/historical-data/src/lib.rs | 5 +- .../src/synch/linear/transaction.rs | 76 +++++++--- 9 files changed, 132 insertions(+), 145 deletions(-) rename core/state-machine/fuzz/fuzz_targets/{fuzz_transactions_with_gc.rs => fuzz_transactions_with_compactness.rs} (52%) diff --git a/core/state-machine/fuzz/Cargo.toml b/core/state-machine/fuzz/Cargo.toml index f5c1129ab314e..b1b482c60adcc 100644 --- a/core/state-machine/fuzz/Cargo.toml +++ b/core/state-machine/fuzz/Cargo.toml @@ -21,9 +21,9 @@ git = "https://github.com/rust-fuzz/libfuzzer-sys.git" members = ["."] [[bin]] -name = "fuzz_transactions_no_gc" +name = "fuzz_transactions_no_compactness" path = "fuzz_targets/fuzz_transactions.rs" [[bin]] -name = "fuzz_transactions_with_gc" -path = "fuzz_targets/fuzz_transactions_with_gc.rs" +name = "fuzz_transactions_with_compactness" +path = "fuzz_targets/fuzz_transactions_with_compactness.rs" diff --git a/core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_gc.rs b/core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs similarity index 52% rename from core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_gc.rs rename to core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs index dda8d261d7b3c..cd80c98a17257 100644 --- a/core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_gc.rs +++ b/core/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs @@ -3,5 +3,5 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - substrate_state_machine_fuzz::fuzz_transactions_then_gc(data) + substrate_state_machine_fuzz::fuzz_transactions_then_compactness(data) }); diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index f8f98bbe6ae13..15b8793ece23d 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -24,7 +24,7 @@ const KEY_SPACE: u8 = 20; /// Size of key, max 255 const VALUE_SPACE: u8 = 50; -fn fuzz_transactions_inner(input: &[u8], check_gc: bool) { +fn fuzz_transactions_inner(input: &[u8], check_compactness: bool) { let mut input_index: usize = 0; let mut overlayed = OverlayedChanges::default(); let mut ref_overlayed = RefOverlayedChanges::default(); @@ -81,17 +81,16 @@ fn fuzz_transactions_inner(input: &[u8], check_gc: bool) { let (check_value, len) = check_values(&overlayed, &ref_overlayed); success &= check_value; - if check_gc { + if check_compactness { let reference_size = ref_overlayed.total_length(); - overlayed.gc(true); let size = overlayed.top_count_keyvalue_pair(); if reference_size != size { println!("inconsistent gc {} {}", size, reference_size); success = false; } - let (check_value, len_gc) = check_values(&overlayed, &ref_overlayed); + let (check_value, len_compactness) = check_values(&overlayed, &ref_overlayed); success &= check_value; - success &= len_gc == len; + success &= len_compactness == len; } ref_overlayed.commit_prospective(); let ref_len = ref_overlayed.committed.len(); @@ -125,7 +124,7 @@ pub fn fuzz_transactions(input: &[u8]) { fuzz_transactions_inner(input, false); } -pub fn fuzz_transactions_then_gc(input: &[u8]) { +pub fn fuzz_transactions_then_compactness(input: &[u8]) { fuzz_transactions_inner(input, true); } diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 150ef66cb51e3..8290dfe4b386d 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -325,8 +325,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historical_data::detached::linear::transaction::{States, TransactionState}; - use historical_data::linear::{History, HistoricalValue}; + use historical_data::synch::linear::transaction::{States, State, History, HistoricalValue}; use super::*; fn prepare_for_build(zero: u64) -> ( @@ -397,39 +396,35 @@ mod test { ]); let changes = OverlayedChanges { changes: OverlayedChangeSet { - states: States::test_vector( - vec![TransactionState::Pending, TransactionState::Pending], - vec![0, 0], - 1, - ), + states: States::test_state(0), top: vec![ (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ (OverlayedValue { value: Some(3u32.encode()), extrinsics: None, - }, 0), + }, State::Committed), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![100], History::from_iter(vec![ (OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) - }, 0), + }, State::Committed), (OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) - }, 1), + }, State::Transaction(0)), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![101], History::from_iter(vec![ (OverlayedValue { value: Some(vec![203]), extrinsics: Some(vec![1].into_iter().collect()) - }, 0), + }, State::Committed), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![103], History::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![0, 1].into_iter().collect()) - }, 1), + }, State::Transaction(0)), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect(), children: vec![ @@ -438,11 +433,11 @@ mod test { (OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) - }, 0), + }, State::Committed), (OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) - }, 1), + }, State::Transaction(0)), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect()), (child_trie_key2, vec![ @@ -450,13 +445,11 @@ mod test { (OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) - }, 1), + }, State::Transaction(0)), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect()), ].into_iter().collect(), }, - operation_from_last_gc: 0, - not_eager_gc: false, changes_trie_config: Some(config.clone()), }; @@ -653,7 +646,7 @@ mod test { (OverlayedValue { value: None, extrinsics: Some(vec![1].into_iter().collect()), - }, 1), + }, State::Transaction(0)), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))); let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index 21613c9db04a2..d8042c65a72ff 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -533,7 +533,8 @@ mod tests { use crate::changes_trie::{Configuration as ChangesTrieConfiguration, InMemoryStorage as InMemoryChangesTrieStorage}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historical_data::linear::{History, HistoricalValue}; + use historical_data::synch::linear::transaction::{History, HistoricalValue}; + use historical_data::synch::linear::transaction::{State}; type TestBackend = InMemory; type TestChangesTrieStorage = InMemoryChangesTrieStorage; @@ -554,18 +555,16 @@ mod tests { (OverlayedValue { value: Some(3u32.encode()), extrinsics: Some(vec![1].into_iter().collect()) - }, 0), + }, State::Committed), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), (vec![1], History::from_iter(vec![ (OverlayedValue { value: Some(vec![100]), extrinsics: Some(vec![1].into_iter().collect()) - }, 0), + }, State::Committed), ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), ].into_iter().collect(), }, - operation_from_last_gc: 0, - not_eager_gc: false, } } diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 69554276a86b9..2a4ed7e5a5639 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -311,19 +311,16 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where CallResult, ) -> CallResult { - let orig_committed = self.overlay.changes.states.committed(); - // disable eager gc to be able to rollback - self.overlay.not_eager_gc = true; + let orig_committed = self.overlay.changes.states.clone(); let (result, was_native, storage_delta, changes_delta) = self.execute_aux( compute_tx, true, native_call.take(), ); - self.overlay.not_eager_gc = false; if was_native { if result.is_ok() { - self.overlay.changes.states.unchecked_rollback_committed(orig_committed); + self.overlay.changes.states = orig_committed; } let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 6fe7f82d8834c..d1e827cf2b361 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -18,13 +18,14 @@ #[cfg(test)] use std::iter::FromIterator; +#[cfg(test)] +use historical_data::synch::linear::transaction::State; use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use historical_data::linear::{History, HistoricalValue}; -use historical_data::detached::linear::transaction::{States}; -use historical_data::detached::DEFAULT_GC_CONF; +use historical_data::synch::linear::transaction::{History, HistoricalValue, States}; +use historical_data::PruneResult; /// The overlayed changes to state to be queried on top of the backend. /// @@ -37,14 +38,6 @@ pub struct OverlayedChanges { /// Changes trie configuration. None by default, but could be installed by the /// runtime if it supports change tries. pub(crate) changes_trie_config: Option, - /// Counter of number of operation between garbage collection. - /// Changing or deleting a value increment this counter by one. - /// An additional cost for data added is added for every n bytes of data. - /// n is currently defined by `DEFAULT_GC_CONF`. - pub(crate) operation_from_last_gc: usize, - /// Can be set to false to ensure we keep pre committed state (in case we want to move - /// back committed cursor). - pub(crate) not_eager_gc: bool, } /// The storage value, used inside OverlayedChanges. @@ -76,7 +69,7 @@ pub struct OverlayedChangeSet { /// Top level storage changes. pub(crate) top: HashMap, History>, /// Child storage changes. - pub(crate) children: HashMap, (HashMap, History>)>, + pub(crate) children: HashMap, HashMap, History>>, } #[cfg(test)] @@ -86,7 +79,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = History::default(); - history.push_unchecked(HistoricalValue { value, index: 0 }); + history.push_unchecked(HistoricalValue { value, index: State::Committed }); history })).collect(); result @@ -117,8 +110,8 @@ fn set_with_extrinsic_inner_overlayed_value( value: Option>, extrinsic_index: u32, ) { - let state = states.num_states() - 1; - if let Some(current) = h_value.get_mut(states) { + let state = states.as_state(); + if let Some(current) = h_value.get_mut() { if current.index == state { current.value.value = value; current.value.extrinsics.get_or_insert_with(Default::default) @@ -128,41 +121,28 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); h_value.push_unchecked(HistoricalValue { + index: state, value: OverlayedValue { value, extrinsics, }, - index: state, }); } } else { let mut extrinsics: Option> = None; extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.push_unchecked(HistoricalValue { + index: state, value: OverlayedValue { value, extrinsics, }, - index: state, }); - } } impl OverlayedChangeSet { - /// Garbage collect. - fn gc(&mut self, prune_commit: bool) { - let states = &self.states; - // retain does change values - self.top.retain(|_, h_value| h_value.get_mut_pruning(states, prune_commit).is_some()); - self.children.retain(|_, m| { - m.retain(|_, h_value| h_value.get_mut_pruning(states, prune_commit).is_some()); - !m.is_empty() - }); - } - /// Whether the change set is empty. pub fn is_empty(&self) -> bool { self.top.is_empty() && self.children.is_empty() @@ -171,11 +151,25 @@ impl OverlayedChangeSet { /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.states.discard_prospective(); + let states = &self.states; + self.top.retain(|_, h_value| states.apply_discard_prospective(h_value) != PruneResult::Cleared); + self.children.retain(|_, m| { + m.retain(|_, h_value| states.apply_discard_prospective(h_value) != PruneResult::Cleared); + !m.is_empty() + }); + self.states.ensure_running(); } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.states.commit_prospective(); + let states = &self.states; + self.top.retain(|_, h_value| states.apply_commit_prospective(h_value) != PruneResult::Cleared); + self.children.retain(|_, m| { + m.retain(|_, h_value| states.apply_commit_prospective(h_value) != PruneResult::Cleared); + !m.is_empty() + }); + self.states.ensure_running(); } /// Create a new transactional layer. @@ -187,11 +181,25 @@ impl OverlayedChangeSet { /// A transaction is always running (states always end with pending). pub fn discard_transaction(&mut self) { self.states.discard_transaction(); + let states = &self.states; + self.top.retain(|_, h_value| states.apply_discard_transaction(h_value) != PruneResult::Cleared); + self.children.retain(|_, m| { + m.retain(|_, h_value| states.apply_discard_transaction(h_value) != PruneResult::Cleared); + !m.is_empty() + }); + self.states.ensure_running(); } /// Commit a transactional layer. pub fn commit_transaction(&mut self) { self.states.commit_transaction(); + let states = &self.states; + self.top.retain(|_, h_value| states.apply_commit_transaction(h_value) != PruneResult::Cleared); + self.children.retain(|_, m| { + m.retain(|_, h_value| states.apply_commit_transaction(h_value) != PruneResult::Cleared); + !m.is_empty() + }); + self.states.ensure_running(); } /// Iterator over current state of a given overlay, including change trie information. @@ -208,7 +216,7 @@ impl OverlayedChangeSet { .into_iter() .flat_map(move |map| map.iter() .filter_map(move |(k, v)| - v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))) + v.get().map(|v| (k.as_slice(), v))) ) } @@ -228,7 +236,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() .filter_map(move |(k, v)| - v.get(self.states.as_ref()) + v.get() .map(|v| (k.as_slice(), v.value.as_ref().map(|v| v.as_slice())))) )) } @@ -241,7 +249,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.to_vec(), child.iter() .filter_map(move |(k, v)| - v.get(self.states.as_ref()) + v.get() .map(|v| (k.to_vec(), v.value.as_ref().map(|v| v.to_vec())))) )) } @@ -253,7 +261,7 @@ impl OverlayedChangeSet { self.children.iter() .map(move |(keyspace, child)| (keyspace.as_slice(), child.iter() .filter_map(move |(k, v)| - v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))) + v.get().map(|v| (k.as_slice(), v))) )) } @@ -264,7 +272,7 @@ impl OverlayedChangeSet { pub(crate) fn top_prospective(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_prospective(&self.states) { + if let Some(v) = v.get_prospective() { result.insert(k.clone(), v.clone()); } } @@ -278,7 +286,7 @@ impl OverlayedChangeSet { pub(crate) fn top_committed(&self) -> HashMap, OverlayedValue> { let mut result = HashMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_committed(&self.states) { + if let Some(v) = v.get_committed() { result.insert(k.clone(), v.clone()); } } @@ -321,7 +329,7 @@ impl OverlayedChanges { /// value has been set. pub fn storage(&self, key: &[u8]) -> Option> { if let Some(overlay_value) = self.changes.top.get(key) { - if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { + if let Some(o_value) = overlay_value.get() { return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } @@ -334,7 +342,7 @@ impl OverlayedChanges { pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.changes.children.get(storage_key) { if let Some(overlay_value) = map.get(key) { - if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { + if let Some(o_value) = overlay_value.get() { return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } @@ -346,7 +354,6 @@ impl OverlayedChanges { /// /// `None` can be used to delete a value specified by the given key. pub fn set_storage(&mut self, key: Vec, value: Option>) { - self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); set_with_extrinsic_overlayed_value(entry, &self.changes.states, value, extrinsic_index); @@ -361,7 +368,6 @@ impl OverlayedChanges { key: Vec, value: Option>, ) { - self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key).or_default(); let entry = map_entry.entry(key).or_default(); @@ -384,7 +390,6 @@ impl OverlayedChanges { let states = &self.changes.states; let map_entry = self.changes.children.entry(storage_key.to_vec()).or_default(); - self.operation_from_last_gc += map_entry.len(); map_entry.values_mut() .for_each(|e| set_with_extrinsic_overlayed_value(e, states, None, extrinsic_index)); } @@ -398,76 +403,48 @@ impl OverlayedChanges { pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); - let mut number_removed = 0; for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - number_removed += 1; set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } - - self.operation_from_last_gc += number_removed; } pub(crate) fn clear_child_prefix(&mut self, storage_key: &[u8], prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); if let Some(child_change) = self.changes.children.get_mut(storage_key) { - let mut number_removed = 0; for (key, entry) in child_change.iter_mut() { if key.starts_with(prefix) { - number_removed += 1; set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } - - self.operation_from_last_gc += number_removed; } } /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.changes.discard_prospective(); - if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { - self.operation_from_last_gc = 0; - self.gc(!self.not_eager_gc); - } } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.changes.commit_prospective(); - if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { - self.operation_from_last_gc = 0; - self.gc(!self.not_eager_gc); - } } /// Create a new transactional layer. pub fn start_transaction(&mut self) { self.changes.start_transaction(); - if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { - self.operation_from_last_gc = 0; - self.gc(false); - } } /// Discard a transactional layer. /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { self.changes.discard_transaction(); - if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { - self.operation_from_last_gc = 0; - self.gc(false); - } } /// Commit a transactional layer. pub fn commit_transaction(&mut self) { self.changes.commit_transaction(); - if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { - self.operation_from_last_gc = 0; - self.gc(false); - } } /// Consume `OverlayedChanges` and take committed set. @@ -477,17 +454,13 @@ impl OverlayedChanges { ){ let top = self.changes.top; let children = self.changes.children; - let committed = self.changes.states.committed(); - let states = self.changes.states.clone(); - let states2 = self.changes.states; ( top.into_iter() - .filter_map(move |(k, v)| v.into_committed(states.as_ref(), committed) + .filter_map(move |(k, v)| v.into_committed() .map(|v| (k, v.value))), children.into_iter().map(move |(sk, v)| { - let states2 = states2.clone(); (sk, v.into_iter() - .filter_map(move |(k, v)| v.into_committed(states2.as_ref(), committed) + .filter_map(move |(k, v)| v.into_committed() .map(|v| (k, v.value)))) }) ) @@ -512,8 +485,6 @@ impl OverlayedChanges { let mut result = OverlayedChanges { changes, changes_trie_config, - operation_from_last_gc: 0, - not_eager_gc: false, }; committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); result.changes.commit_prospective(); @@ -556,13 +527,6 @@ impl OverlayedChanges { } result } - - /// costy garbage collection of unneeded memory from - /// key values. Eager set to true will remove more - /// key value but allows more costy memory changes. - pub fn gc(&mut self, eager: bool) { - self.changes.gc(eager); - } } #[cfg(test)] diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index bc975439cb207..d5ff6069e9a6e 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -27,7 +27,7 @@ pub mod linear; /// An entry at a given history index. #[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test"), derive(PartialEq))] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] pub struct HistoricalValue { /// The stored value. pub value: V, @@ -51,7 +51,8 @@ impl HistoricalValue { } } -#[cfg_attr(any(test, feature = "test"), derive(PartialEq, Debug))] +#[derive(PartialEq)] +#[cfg_attr(any(test, feature = "test"), derive(Debug))] /// Prunning result to be able to proceed /// with further update if the value needs it. pub enum PruneResult { diff --git a/core/utils/historical-data/src/synch/linear/transaction.rs b/core/utils/historical-data/src/synch/linear/transaction.rs index 1621b30d47b9f..922be8301aee5 100644 --- a/core/utils/historical-data/src/synch/linear/transaction.rs +++ b/core/utils/historical-data/src/synch/linear/transaction.rs @@ -32,18 +32,24 @@ use rstd::vec::Vec; use crate::PruneResult; /// Global state is a simple counter to the current overlay layer index. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct States(usize); impl Default for States { fn default() -> Self { - States(0) + // we default to 1 to be able to discard this transaction. + States(1) } } impl States { + /// Get corresponding current state. + pub fn as_state(&self) -> State { + State::Transaction(self.0) + } + + /// Build any state for testing only. #[cfg(any(test, feature = "test-helpers"))] pub fn test_state( @@ -52,13 +58,21 @@ impl States { States(current_layer_number) } + /// By default the current transaction is at least 1. + /// 0 can be use only to apply transaction change, in + /// this case transaction need to be restored to a valid + /// state afterward. + pub fn ensure_running(&mut self) { + if self.0 == 0 { + self.0 = 1; + } + } + /// Discard prospective changes to state. /// It does not reverts actual values. /// A subsequent synchronisation of stored values is needed. pub fn discard_prospective(&mut self) { - if self.0 > 0 { - self.0 -= 1; - } + self.0 = 1; } /// Update a value to a new prospective. @@ -80,9 +94,7 @@ impl States { /// Commit prospective changes to state. /// A subsequent synchronisation of stored values is needed. pub fn commit_prospective(&mut self) { - if self.0 > 0 { - self.0 -= 1; - } + self.0 = 1; } /// Update a value to a new prospective. @@ -157,38 +169,37 @@ impl States { /// Multiple discard can be applied at the same time. /// Returns true if value is still needed. pub fn apply_commit_transaction(&self, value: &mut History) -> PruneResult { - let init_len = value.0.len(); + let target_overlay = if self.0 > 0 { + self.0 + } else { + // `ensure_running` will put state back to one after + 1 + }; let mut new_value = None; for i in (0 .. value.0.len()).rev() { if let HistoricalValue { value: _, index: State::Transaction(ix), } = value.0[i] { - if ix > self.0 { + if ix > target_overlay { if let Some(v) = value.0.pop() { if new_value.is_none() { new_value = Some(v.value); } } - } else if ix == self.0 && new_value.is_some() { + } else if ix == target_overlay && new_value.is_some() { let _ = value.0.pop(); } else { break } } else { break } } - let result = if value.0.len() == 0 { - PruneResult::Cleared - } else if value.0.len() != init_len { - PruneResult::Changed - } else { - PruneResult::Unchanged - }; if let Some(new_value) = new_value { value.0.push(HistoricalValue { value: new_value, - index: State::Transaction(self.0), + index: State::Transaction(target_overlay), }); + return PruneResult::Changed; } - result + PruneResult::Unchanged } } @@ -200,6 +211,7 @@ pub enum State { Committed, Transaction(usize), } + impl State { fn transaction_index(&self) -> Option { if let &State::Transaction(ix) = self { @@ -224,6 +236,22 @@ impl Default for History { } impl History { + + /// Get current number of stored historical values. + pub fn len(&self) -> usize { + self.0.len() + } + + #[cfg(any(test, feature = "test-helpers"))] + /// Create an history from an existing history. + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = History::default(); + for v in input { + history.push_unchecked(v); + } + history + } + /// Set a value, it uses a global state as parameter. pub fn set(&mut self, states: &States, value: V) { if let Some(v) = self.0.last_mut() { @@ -242,6 +270,12 @@ impl History { }); } + /// Push a value without checking if it can overwrite the current + /// state. + pub fn push_unchecked(&mut self, item: HistoricalValue) { + self.0.push(item); + } + /// Access to the latest pending value. pub fn get(&self) -> Option<&V> { self.0.last().map(|h| &h.value) From e265ad144c6e5b2446db18b92170c4d1ba6846b1 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Wed, 13 Nov 2019 18:25:31 +0100 Subject: [PATCH 077/129] New implementation accounts for a 17 and 20% lost on existing bench. --- core/state-machine/fuzz/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state-machine/fuzz/src/lib.rs b/core/state-machine/fuzz/src/lib.rs index 15b8793ece23d..cf1ee726d73b8 100644 --- a/core/state-machine/fuzz/src/lib.rs +++ b/core/state-machine/fuzz/src/lib.rs @@ -240,8 +240,8 @@ impl RefOverlayedChanges { #[test] fn previous_fuzzed_error() { let inputs = [ - vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], + vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], vec![0x2,0xee,0xee,0x12,0x2,0x16,0x67,0xee,0xee,0xee,], vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], From 9904c4f952a9026300f89c33d7425bb2601ab29f Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 Nov 2019 11:39:46 +0100 Subject: [PATCH 078/129] Remove unuse code. Test failings. --- core/sr-io/src/lib.rs | 2 +- core/sr-io/with_std.rs | 2 +- core/state-machine/src/changes_trie/build.rs | 2 +- core/state-machine/src/ext.rs | 3 +- core/state-machine/src/lib.rs | 4 +- core/state-machine/src/overlayed_changes.rs | 4 +- core/test-runtime/src/lib.rs | 14 +- .../src/detached/linear/mod.rs | 19 - .../src/detached/linear/transaction.rs | 496 ------------------ .../utils/historical-data/src/detached/mod.rs | 118 ----- core/utils/historical-data/src/lib.rs | 10 +- core/utils/historical-data/src/linear.rs | 102 ---- .../historical-data/src/synch/linear/mod.rs | 19 - core/utils/historical-data/src/synch/mod.rs | 21 - ...saction.rs => synch_linear_transaction.rs} | 5 + 15 files changed, 22 insertions(+), 799 deletions(-) delete mode 100644 core/utils/historical-data/src/detached/linear/mod.rs delete mode 100644 core/utils/historical-data/src/detached/linear/transaction.rs delete mode 100644 core/utils/historical-data/src/detached/mod.rs delete mode 100644 core/utils/historical-data/src/linear.rs delete mode 100644 core/utils/historical-data/src/synch/linear/mod.rs delete mode 100644 core/utils/historical-data/src/synch/mod.rs rename core/utils/historical-data/src/{synch/linear/transaction.rs => synch_linear_transaction.rs} (97%) diff --git a/core/sr-io/src/lib.rs b/core/sr-io/src/lib.rs index 2052ec22600c0..b57b649fb208b 100644 --- a/core/sr-io/src/lib.rs +++ b/core/sr-io/src/lib.rs @@ -148,7 +148,7 @@ export_api! { /// Commit a transactional layer. fn storage_commit_transaction(); - + /// A trie root formed from the iterated items. fn blake2_256_trie_root(input: Vec<(Vec, Vec)>) -> H256; diff --git a/core/sr-io/with_std.rs b/core/sr-io/with_std.rs index 1b89264747443..2ec0ecb78e368 100644 --- a/core/sr-io/with_std.rs +++ b/core/sr-io/with_std.rs @@ -180,7 +180,7 @@ impl StorageApi for () { ext.storage_commit_transaction() ); } - + fn blake2_256_trie_root(input: Vec<(Vec, Vec)>) -> H256 { Layout::::trie_root(input) } diff --git a/core/state-machine/src/changes_trie/build.rs b/core/state-machine/src/changes_trie/build.rs index 8290dfe4b386d..8f02e54da4130 100644 --- a/core/state-machine/src/changes_trie/build.rs +++ b/core/state-machine/src/changes_trie/build.rs @@ -325,7 +325,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historical_data::synch::linear::transaction::{States, State, History, HistoricalValue}; + use historical_data::synch_linear_transaction::{States, State, History, HistoricalValue}; use super::*; fn prepare_for_build(zero: u64) -> ( diff --git a/core/state-machine/src/ext.rs b/core/state-machine/src/ext.rs index d8042c65a72ff..52fe20e012bf8 100644 --- a/core/state-machine/src/ext.rs +++ b/core/state-machine/src/ext.rs @@ -533,8 +533,7 @@ mod tests { use crate::changes_trie::{Configuration as ChangesTrieConfiguration, InMemoryStorage as InMemoryChangesTrieStorage}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use historical_data::synch::linear::transaction::{History, HistoricalValue}; - use historical_data::synch::linear::transaction::{State}; + use historical_data::synch_linear_transaction::{History, HistoricalValue, State}; type TestBackend = InMemory; type TestChangesTrieStorage = InMemoryChangesTrieStorage; diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 2a4ed7e5a5639..515a97b212d2b 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -319,9 +319,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where ); if was_native { - if result.is_ok() { - self.overlay.changes.states = orig_committed; - } + self.overlay.changes.states = orig_committed; let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, false, diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index d1e827cf2b361..3f9ca585d7d76 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -19,12 +19,12 @@ #[cfg(test)] use std::iter::FromIterator; #[cfg(test)] -use historical_data::synch::linear::transaction::State; +use historical_data::synch_linear_transaction::State; use std::collections::{HashMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; -use historical_data::synch::linear::transaction::{History, HistoricalValue, States}; +use historical_data::synch_linear_transaction::{History, HistoricalValue, States}; use historical_data::PruneResult; /// The overlayed changes to state to be queried on top of the backend. diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 3a5b548adaa5c..c66c79ce8595b 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -916,23 +916,25 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { } fn test_historical_data() { - let mut states = historical_data::detached::linear::transaction::States::default(); - let mut value = historical_data::linear::History::default(); - if value.get(states.as_ref()) != None { + let mut states = historical_data::synch_linear_transaction::States::default(); + let mut value = historical_data::synch_linear_transaction::History::default(); + if value.get() != None { panic!("Got a value for empty data"); } value.set(&states, 42u64); states.start_transaction(); - if value.get(states.as_ref()) != Some(&42) { + if value.get() != Some(&42) { panic!("Got a wrong result accessing a one element data"); } value.set(&states, 43u64); - if value.get(states.as_ref()) != Some(&43) { + if value.get() != Some(&43) { panic!("Got a wrong result accessing a two element data"); } states.discard_transaction(); - if value.get(states.as_ref()) != Some(&42) { + states.apply_discard_transaction(&mut value); + states.ensure_running(); + if value.get() != Some(&42) { panic!("Got a wrong result accessing a one element data after a discard"); } } diff --git a/core/utils/historical-data/src/detached/linear/mod.rs b/core/utils/historical-data/src/detached/linear/mod.rs deleted file mode 100644 index 736360ce584a9..0000000000000 --- a/core/utils/historical-data/src/detached/linear/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Linear detached historical. - -pub mod transaction; diff --git a/core/utils/historical-data/src/detached/linear/transaction.rs b/core/utils/historical-data/src/detached/linear/transaction.rs deleted file mode 100644 index 8d02207949478..0000000000000 --- a/core/utils/historical-data/src/detached/linear/transaction.rs +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Transactional layer implementation. -//! -//! General principle of historical data (see module design)applies here. -//! Therefore we got a state and values with an history dependant on -//! a state. -//! -//! This module is defining a linear transactional state. -//! -//! ## state operation -//! -//! The state history contains multiple possible states `TransactionStates` and -//! a committed index. -//! -//! There is two pending state, both indicate a valid value, but on also indicate -//! the start of a transactional window. -//! Dropping a transaction just invalidate all state from the start of the last -//! transactional window. -//! Committing a transaction fused it transaction window with the previous transaction -//! window (changing a `TxPending` with a `Pending` state). -//! -//! The committed index is lie a upper transaction state and is considered as the start -//! of a transaction, but cannot be dropped with the same operation. -//! If dropped, all state after this index are dropped. -//! If committed its index is updated and all state prior cannot be dropped. -//! -//! ## value operation -//! -//! Here access to latest state value is a reverse iteration over history of value and -//! matching state, up to the first pending state. -//! -//! On mutable access, terminal dropped state or unneeded state values (values in a same -//! transactional window) are dropped. -//! This allows a clean state up to the latest transactional window at a small cost. -//! -//! ** Update only occurs on a latest state **, therefore state primitives should not allow -//! access to some subset of the state (full state is used as parameter when for query a simple -//! slice is used). -//! -//! # Usage -//! -//! All the primitives for a value expect a reference to the state used to build -//! the value. ** Wrong state usage results in undefined behavior. ** - - -use rstd::vec::Vec; -use rstd::vec; -use crate::linear::{HistoricalValue, History}; - -#[derive(Debug, Clone, Eq, PartialEq, Copy)] -/// State of a transactional layer. -pub enum TransactionState { - /// Data is under change and can still be dropped. - Pending, - /// Same as pending but does count as a transaction start. - TxPending, - /// Data pointing to this indexed historic state should - /// not be returned and can be removed. - Dropped, -} - -/// States is both an indexed state to query values with history -/// and a committed index that indicates a point in time where -/// we cannot drop transaction layer. -/// Committed index is starting at 1, if it is 0 then there is no -/// committed index and all layer can be dropped. -/// There is a implicit pending state which is equal to the length -/// of this history. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct States { - history: Vec, - // Keep track of the transaction positions. - // This is redundant with `history`. - // This is only use to precalculate the result - // of a backward iteration to find the start - // of a transactional window (function - // `transaction_start_windows`). - // - // At many place we could use this instead - // of the `TxPending` state, but `TxPending` - // state is favored, except for `start_windows` - // case. - start_transaction_window: Vec, - commit: usize, -} - -impl Default for States { - fn default() -> Self { - States { - history: vec![TransactionState::Pending], - start_transaction_window: vec![0], - commit: 0, - } - } -} - -impl States { - /// Get reference of state, that is enough - /// information to query historical - /// data. - pub fn as_ref(&self) -> &[TransactionState] { - self.history.as_ref() - } - - /// Current number of inner states. - pub fn num_states(&self) -> usize { - self.history.len() - } - - /// Get index of committed layer. - pub fn committed(&self) -> usize { - self.commit - } - - /// Allow to rollback to a previous committed - /// index. - /// This can only work if there was no eager - /// garbage collection. - pub fn unchecked_rollback_committed(&mut self, old_committed: usize) { - self.commit = old_committed; - self.discard_prospective(); - } - - /// Build any state for testing only. - #[cfg(any(test, feature = "test-helpers"))] - pub fn test_vector( - history: Vec, - start_transaction_window: Vec, - commit: usize, - ) -> Self { - States { history, start_transaction_window, commit } - } - - /// Discard prospective changes to state. - /// That is revert all transaction up to the committed index. - pub fn discard_prospective(&mut self) { - for i in self.commit .. self.history.len() { - self.history[i] = TransactionState::Dropped; - self.start_transaction_window[i] = self.commit; - } - self.history.push(TransactionState::Pending); - self.start_transaction_window.push(self.commit); - } - - /// Commit prospective changes to state. - pub fn commit_prospective(&mut self) { - self.commit = self.history.len(); - self.history.push(TransactionState::Pending); - for i in 0..self.history.len() - 1 { - self.start_transaction_window[i] = self.commit; - } - self.start_transaction_window.push(self.commit); - } - - /// Create a new transactional layer. - pub fn start_transaction(&mut self) { - self.start_transaction_window.push(self.history.len()); - self.history.push(TransactionState::TxPending); - } - - /// Discard a transactional layer. - /// A transaction is always running (history always end with pending). - pub fn discard_transaction(&mut self) { - let mut i = self.history.len(); - while i > self.commit { - i -= 1; - match self.history[i] { - TransactionState::Dropped => (), - TransactionState::Pending => { - self.history[i] = TransactionState::Dropped; - }, - TransactionState::TxPending => { - self.history[i] = TransactionState::Dropped; - break; - }, - } - } - self.history.push(TransactionState::Pending); - self.start_transaction_window.truncate(i); - let previous = self.start_transaction_window.last() - .cloned().unwrap_or(self.commit); - self.start_transaction_window.resize(self.history.len(), previous); - } - - /// Commit a transactional layer. - pub fn commit_transaction(&mut self) { - let mut i = self.history.len(); - while i > self.commit { - i -= 1; - match self.history[i] { - TransactionState::Pending - | TransactionState::Dropped => (), - TransactionState::TxPending => { - self.history[i] = TransactionState::Pending; - break; - }, - } - } - self.history.push(TransactionState::Pending); - self.start_transaction_window.truncate(i); - let previous = self.start_transaction_window.last() - .cloned().unwrap_or(self.commit); - self.start_transaction_window.resize(self.history.len(), previous); - } -} - -#[inline] -/// Get previous index of pending state. -/// -/// Used to say if it is possible to drop a committed transaction -/// state value. -/// Committed index is seen as a transaction state. -pub fn transaction_start_windows(states: &States, from: usize) -> usize { - states.start_transaction_window[from] -} - -impl History { - fn get_pending_unchecked(&self, states: &[TransactionState], history_index: usize) - -> Option> { - let HistoricalValue { value, index } = self.get_unchecked(history_index); - match states[index] { - TransactionState::Dropped => (), - TransactionState::Pending - | TransactionState::TxPending => - return Some(HistoricalValue { value, index }), - } - None - } - - /// Set a value, it uses a state history as parameter. - /// - /// This method uses `get_mut` and does remove pending - /// dropped value. - pub fn set(&mut self, states: &States, value: V) { - let last_state_index = states.num_states() - 1; - if let Some(v) = self.get_mut(states) { - if v.index == last_state_index { - *v.value = value; - return; - } - debug_assert!(v.index < last_state_index, "History expects \ - only new values at the latest state"); - } - self.push_unchecked(HistoricalValue { - value, - index: last_state_index, - }); - } - - /// Access to the latest pending value (non dropped state). - /// When possible please prefer `get_mut` as it can free - /// some memory. - pub fn get(&self, states: &[TransactionState]) -> Option<&V> { - let self_len = self.len(); - if self_len == 0 { - return None; - } - debug_assert!(states.len() >= self_len); - for index in (0 .. self_len).rev() { - if let Some(h) = self.get_pending_unchecked(states, index) { - return Some(h.value); - } - } - None - } - - /// Get latest value, consuming the historical data. - pub fn into_pending(mut self, states: &[TransactionState]) -> Option { - let self_len = self.len(); - if self_len == 0 { - return None; - } - debug_assert!(states.len() >= self_len); - for index in (0 .. self_len).rev() { - if self.get_pending_unchecked(states, index).is_some() { - self.truncate(index + 1); - return self.pop().map(|v| v.value); - } - } - None - } - - - #[cfg(any(test, feature = "test-helpers"))] - pub fn get_prospective(&self, states: &States) -> Option<&V> { - let self_len = self.len(); - if self_len == 0 { - return None; - } - debug_assert!(states.history.len() >= self_len); - for index in (states.commit .. self_len).rev() { - if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { - return Some(h.value); - } - } - None - } - - #[cfg(any(test, feature = "test-helpers"))] - pub fn get_committed(&self, states: &States) -> Option<&V> { - let self_len = self.len(); - if self_len == 0 { - return None; - } - debug_assert!(states.history.len() >= self_len); - for index in (0 .. self_len).rev() { - if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { - if h.index < states.commit { - return Some(h.value); - } - } - } - None - } - - pub fn into_committed(mut self, states: &[TransactionState], committed: usize) -> Option { - let self_len = self.len(); - if self_len == 0 { - return None; - } - debug_assert!(states.len() >= self_len); - for index in (0 .. self_len).rev() { - if let Some(h) = self.get_pending_unchecked(states, index) { - if h.index < committed { - self.truncate(index + 1); - return self.pop().map(|v| v.value); - } - } - } - None - } - - /// Access to latest pending value (non dropped state). - /// - /// This method removes latest dropped and merged values, - /// only keeping the latest valid value. - /// - /// Returns mutable latest pending value. - pub fn get_mut( - &mut self, - states: &States, - ) -> Option> { - let self_len = self.len(); - if self_len == 0 { - return None; - } - debug_assert!(states.history.len() >= self_len); - let mut result = None; - let mut start_transaction_window = usize::max_value(); - let mut previous_switch = None; - for index in (0 .. self_len).rev() { - let state_index = self.get_unchecked(index).index; - match states.history[state_index] { - TransactionState::TxPending => { - if state_index >= start_transaction_window { - previous_switch = Some(index); - } else { - if result.is_none() { - result = Some((index, state_index)); - } - } - break; - }, - TransactionState::Pending => { - if state_index >= start_transaction_window { - previous_switch = Some(index); - } else { - if result.is_none() { - result = Some((index, state_index)); - start_transaction_window = transaction_start_windows(states, state_index); - } else { - break; - } - } - }, - TransactionState::Dropped => (), - } - } - self.drop_last_values(result, previous_switch, 0) - } - - - /// Method to prune regarding a full state. - /// It also returns the last value as mutable. - /// Internally it acts like `get_mut` with an - /// additional cleaning capacity to clean committed - /// state if `prune_to_commit` is set to true. - pub fn get_mut_pruning( - &mut self, - states: &States, - prune_to_commit: bool, - ) -> Option> { - let self_len = self.len(); - if self_len == 0 { - return None; - } - let mut prune_index = 0; - debug_assert!(states.history.len() >= self_len); - let mut result = None; - let mut start_transaction_window = usize::max_value(); - let mut previous_switch = None; - for index in (0 .. self_len).rev() { - let state_index = self.get_unchecked(index).index; - match states.history[state_index] { - state @ TransactionState::TxPending - | state @ TransactionState::Pending => { - if state_index < states.commit && index > prune_index { - prune_index = index; - } - - if state_index >= start_transaction_window { - previous_switch = Some(index); - } else { - if result.is_none() { - result = Some((index, state_index)); - if state == TransactionState::Pending { - start_transaction_window = transaction_start_windows(states, state_index); - } - } else { - if prune_to_commit { - if state_index < states.commit { - break; - } - } else { - break; - } - } - } - }, - TransactionState::Dropped => (), - } - } - let deleted = if prune_to_commit && prune_index > 0 && result.is_some() { - self.remove_until(prune_index); - prune_index - } else { - 0 - }; - self.drop_last_values(result, previous_switch, deleted) - } - - // Function used to remove all last values for `get_mut` and - // `get_mut_pruning`. - // - // It expects the `result` of the iteration that lookup - // for the latest non dropped value (coupled with its state - // index). That is to remove terminal dropped states. - // - // It also does remove values on committed state that are - // not needed (before another committed value). - // `previous_switch` is the index to the first unneeded value. - // - // An index offset is here in case some content was `deleted` before - // this function call. - fn drop_last_values( - &mut self, - result: Option<(usize, usize)>, - previous_switch: Option, - deleted: usize, - ) -> Option> { - if let Some((index, state_index)) = result { - if index + 1 - deleted < self.len() { - self.truncate(index + 1 - deleted); - } - if let Some(switch_index) = previous_switch { - if let Some(mut value) = self.pop() { - self.truncate(switch_index - deleted); - value.index = state_index; - self.push_unchecked(value); - } - Some((self.mut_ref(switch_index - deleted), state_index).into()) - } else { - Some((self.mut_ref(index - deleted), state_index).into()) - } - } else { - self.0.clear(); - None - } - } -} diff --git a/core/utils/historical-data/src/detached/mod.rs b/core/utils/historical-data/src/detached/mod.rs deleted file mode 100644 index b5e701f20b58d..0000000000000 --- a/core/utils/historical-data/src/detached/mod.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! # Design -//! -//! The historical storage is loosely related to a global detached -//! state. -//! -//! Two elements are used, a global state and multiple historical -//! values. -//! Multiple states for an historical value only make sense if the states -//! got a common basis, that is using partial reference (subsets of a -//! unique global set). -//! -//! ## state operation -//! -//! The state is an historic of state and can be seen as an index -//! for querying or updated the value. -//! State operations, are only changing the state, but not the -//! related stored value (allowing low cost update). -//! If the state cannot modify the existing values, it means that -//! the state cannot change its indexing, it can change a state for -//! an index but cannot remove it, thus it is an append only structure. -//! -//! An operation on a value relates to the state by the last index of the state -//! for the value operation (field `index` of `HistoricalValue`). -//! -//! Therefore the state can be used trimed from its latest index to query a -//! previous state. -//! -//! It should be possible to mutably reduce the state under the assumption -//! that all linked values where pruned accordingly (or by executing an -//! index adjustment on all values atomically). -//! Generally this is not a good idea, as it defeats the purpose of this crate: -//! state size is marginal in comparison of values size. -//! Additionally values query should be very moderatly impacted by the state size -//! (for linear state it is not, but for a tree state the indexing between two branches -//! involves a small additional cost to jump between branch). -//! -//! ## value operation -//! -//! We said before that state operations do not update values, so the value operations -//! will always use the state (or part of the state) as a parameter in order -//! return a different value depending on this state parameter. -//! Operation are simply update of value and queries, but to allow query at different -//! historic state, the values (`HistoricalValue`) do keep reference to all possible values. -//! -//! The crate favors light state operation that do not force update of values, -//! but sometime limited updates on values mutable access can be done (if the cost is -//! close to query cost). -//! To avoid memory exhaustion, a costy manual pruning for value corresponding to dead state -//! is provided (usually call `get_mut_pruning`). -//! It can be use to set in place a garbage collection. - - -pub mod linear; - -/// A default sample configuration to manage garbage collection -/// triggering. -pub const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { - trigger_transaction_gc: 100_000, - trigger_commit_gc: 10_000, - add_content_size_unit: 64, -}; - -/// Garbage collection configuration. -/// It is designed to listen on two different operation, transaction -/// and commit. -/// It match transcational semantic with transaction for transaction -/// operation and commit for prospective operation. -pub struct GCConfiguration { - /// Treshold in number of operation before running a garbage collection. - /// - /// Should be same as `TRIGGER_COMMIT_GC` or higher - /// (we most likely do not want lower as transaction are - /// possibly more frequent than commit). - pub trigger_transaction_gc: usize, - - /// Treshold in number of operation before running a garbage colletion - /// on a commit operation. - /// - /// We may want a lower value than for a transaction, even - /// a 1 if we want to do it between every operation. - pub trigger_commit_gc: usize, - - /// Used to count big content as multiple operations. - /// This is a number of octet. - /// Set to 0 to ignore. - pub add_content_size_unit: usize, -} - -impl GCConfiguration { - /// Cost depending on value if any. - pub fn operation_cost(&self, val: Option<&rstd::vec::Vec>) -> usize { - let additional_cost = if self.add_content_size_unit > 0 { - if let Some(s) = val.as_ref() { - s.len() / self.add_content_size_unit - } else { - 0 - } - } else { 0 }; - 1 + additional_cost - } - -} diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index d5ff6069e9a6e..72d3864a8d0aa 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -16,14 +16,12 @@ //! Data storage containing multiple state for a value. //! This is use to store historical information for an item, -//! and does have to include any kind of proof. +//! and does have to include any proof. #![cfg_attr(not(feature = "std"), no_std)] -pub mod synch; -pub mod detached; -pub mod linear; +pub mod synch_linear_transaction; /// An entry at a given history index. #[derive(Debug, Clone)] @@ -42,10 +40,6 @@ impl From<(V, I)> for HistoricalValue { } impl HistoricalValue { - fn as_ref(&self) -> HistoricalValue<&V, I> { - HistoricalValue { value: &self.value, index: self.index.clone() } - } - fn as_mut(&mut self) -> HistoricalValue<&mut V, I> { HistoricalValue { value: &mut self.value, index: self.index.clone() } } diff --git a/core/utils/historical-data/src/linear.rs b/core/utils/historical-data/src/linear.rs deleted file mode 100644 index 5fdd090cc6aba..0000000000000 --- a/core/utils/historical-data/src/linear.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Linear historical state container. - - -/// An entry at a given history height. -pub type HistoricalValue = crate::HistoricalValue; - -/// Array like buffer for in memory storage. -/// By in memory we expect that this will -/// not required persistence and is not serialized. -type MemoryOnly = smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>; - -/// Size of preallocated history per element. -/// Currently at two for committed and prospective only. -/// It means that using transaction in a module got a direct allocation cost. -const ALLOCATED_HISTORY: usize = 2; - -/// History of value that are related to a state history (eg field `history` of -/// an `OverlayedChangeSet`). -/// -/// Values are always paired with a state history index. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct History(pub(crate) MemoryOnly); - -impl Default for History { - fn default() -> Self { - History(Default::default()) - } -} - -// Following implementation are here to isolate -// buffer specific functions. -impl History { - pub(crate) fn get_unchecked(&self, index: usize) -> HistoricalValue<&V> { - self.0[index].as_ref() - } - - #[cfg(any(test, feature = "test-helpers"))] - /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator>) -> Self { - let mut history = History::default(); - for v in input { - history.push_unchecked(v); - } - history - } - - /// Current number of inner states. - pub fn len(&self) -> usize { - self.0.len() - } - - pub(crate) fn truncate(&mut self, index: usize) { - self.0.truncate(index) - } - - pub(crate) fn remove_until(&mut self, index: usize) { - if index > 0 { - if self.0.spilled() { - let owned = rstd::mem::replace(&mut self.0, Default::default()); - self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); - } else { - for i in (0..index).rev() { - self.0.remove(i); - } - } - } - } - - pub(crate) fn pop(&mut self) -> Option> { - self.0.pop() - } - - /// Append without checking if a value already exist. - /// If a value already exists, the history will be broken. - /// This method shall only be call after a `get_mut` where - /// the returned index indicate that a `set` will result - /// in appending a value. - pub fn push_unchecked(&mut self, value: HistoricalValue) { - self.0.push(value) - } - - pub(crate) fn mut_ref(&mut self, index: usize) -> &mut V { - &mut self.0[index].value - } -} diff --git a/core/utils/historical-data/src/synch/linear/mod.rs b/core/utils/historical-data/src/synch/linear/mod.rs deleted file mode 100644 index e9a35ef64b51b..0000000000000 --- a/core/utils/historical-data/src/synch/linear/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Linear arrangement of historical data. - -pub mod transaction; diff --git a/core/utils/historical-data/src/synch/mod.rs b/core/utils/historical-data/src/synch/mod.rs deleted file mode 100644 index f9e0425b44898..0000000000000 --- a/core/utils/historical-data/src/synch/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Synch module contains historical data that do not require -//! a separated state for most operation. Global state operation -//! must therefore apply on all values with history synchronously. - -pub mod linear; diff --git a/core/utils/historical-data/src/synch/linear/transaction.rs b/core/utils/historical-data/src/synch_linear_transaction.rs similarity index 97% rename from core/utils/historical-data/src/synch/linear/transaction.rs rename to core/utils/historical-data/src/synch_linear_transaction.rs index 922be8301aee5..41f46653e57ac 100644 --- a/core/utils/historical-data/src/synch/linear/transaction.rs +++ b/core/utils/historical-data/src/synch_linear_transaction.rs @@ -18,6 +18,11 @@ //! support. //! //! +//! Described as a synchronized module as it contains historical +//! data that do not require a separated state for most operation. +//! Global state operation must therefore apply on all local values +//! with history synchronously. +//! //! # Global state //! //! The only global state is a counter of overlayed transaction layer. From 5f82305131f5a510a8e194cd5c3ddf1e3927213e Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 Nov 2019 15:35:01 +0100 Subject: [PATCH 079/129] Fix broken tests (obviously cloning only state do not work anymore). --- core/state-machine/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state-machine/src/lib.rs b/core/state-machine/src/lib.rs index 09c8431a7cbb0..0d9a008334ba7 100644 --- a/core/state-machine/src/lib.rs +++ b/core/state-machine/src/lib.rs @@ -311,7 +311,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where CallResult, ) -> CallResult { - let orig_committed = self.overlay.changes.states.clone(); + let orig_committed = self.overlay.changes.clone(); let (result, was_native, storage_delta, changes_delta) = self.execute_aux( compute_tx, true, @@ -319,7 +319,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where ); if was_native { - self.overlay.changes.states = orig_committed; + self.overlay.changes = orig_committed; let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, false, From 494ad42692575acc7202542b197cf883d6a6198a Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 Nov 2019 15:53:39 +0100 Subject: [PATCH 080/129] Removing trailing spaces. --- core/state-machine/src/overlayed_changes.rs | 4 ++-- core/test-runtime/src/lib.rs | 2 +- .../historical-data/src/synch_linear_transaction.rs | 9 +++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 3f9ca585d7d76..91dd14ed35927 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -446,7 +446,7 @@ impl OverlayedChanges { pub fn commit_transaction(&mut self) { self.changes.commit_transaction(); } - + /// Consume `OverlayedChanges` and take committed set. pub fn into_committed(self) -> ( impl Iterator, Option>)>, @@ -725,7 +725,7 @@ mod tests { assert!(overlayed.storage(&key).is_none()); // discard transaction similar to discard prospective if no transaction. - + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index a54d49a017b4a..2f48468e94548 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -919,7 +919,7 @@ fn test_historical_data() { if value.get() != None { panic!("Got a value for empty data"); } - + value.set(&states, 42u64); states.start_transaction(); if value.get() != Some(&42) { diff --git a/core/utils/historical-data/src/synch_linear_transaction.rs b/core/utils/historical-data/src/synch_linear_transaction.rs index 41f46653e57ac..939265b227efa 100644 --- a/core/utils/historical-data/src/synch_linear_transaction.rs +++ b/core/utils/historical-data/src/synch_linear_transaction.rs @@ -27,7 +27,7 @@ //! //! The only global state is a counter of overlayed transaction layer. //! Committing or discarding a layer must use this counter. -//! +//! //! # Local state //! //! Local state is either a committed state (this is a single first independant level @@ -36,10 +36,10 @@ use rstd::vec::Vec; use crate::PruneResult; -/// Global state is a simple counter to the current overlay layer index. +/// Global states is a simple counter to the current overlay layer index. #[derive(Debug, Clone, PartialEq, Eq)] pub struct States(usize); - + impl Default for States { fn default() -> Self { // we default to 1 to be able to discard this transaction. @@ -74,7 +74,7 @@ impl States { } /// Discard prospective changes to state. - /// It does not reverts actual values. + /// It does not reverts actual values. /// A subsequent synchronisation of stored values is needed. pub fn discard_prospective(&mut self) { self.0 = 1; @@ -226,6 +226,7 @@ impl State { } } } + /// An entry at a given history height. pub type HistoricalValue = crate::HistoricalValue; From f7b959472359a9d343674046dc776959801fbc41 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 Nov 2019 16:13:58 +0100 Subject: [PATCH 081/129] rename 'ensure_runing' to 'finalize_discard' as it is only really needed only for discard_transaction. --- core/state-machine/src/overlayed_changes.rs | 5 +--- core/test-runtime/src/lib.rs | 2 +- .../src/synch_linear_transaction.rs | 25 +++++++------------ 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 91dd14ed35927..508d39b7aca5c 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -157,7 +157,6 @@ impl OverlayedChangeSet { m.retain(|_, h_value| states.apply_discard_prospective(h_value) != PruneResult::Cleared); !m.is_empty() }); - self.states.ensure_running(); } /// Commit prospective changes to state. @@ -169,7 +168,6 @@ impl OverlayedChangeSet { m.retain(|_, h_value| states.apply_commit_prospective(h_value) != PruneResult::Cleared); !m.is_empty() }); - self.states.ensure_running(); } /// Create a new transactional layer. @@ -187,7 +185,7 @@ impl OverlayedChangeSet { m.retain(|_, h_value| states.apply_discard_transaction(h_value) != PruneResult::Cleared); !m.is_empty() }); - self.states.ensure_running(); + self.states.finalize_discard(); } /// Commit a transactional layer. @@ -199,7 +197,6 @@ impl OverlayedChangeSet { m.retain(|_, h_value| states.apply_commit_transaction(h_value) != PruneResult::Cleared); !m.is_empty() }); - self.states.ensure_running(); } /// Iterator over current state of a given overlay, including change trie information. diff --git a/core/test-runtime/src/lib.rs b/core/test-runtime/src/lib.rs index 2f48468e94548..16da867a90547 100644 --- a/core/test-runtime/src/lib.rs +++ b/core/test-runtime/src/lib.rs @@ -931,7 +931,7 @@ fn test_historical_data() { } states.discard_transaction(); states.apply_discard_transaction(&mut value); - states.ensure_running(); + states.finalize_discard(); if value.get() != Some(&42) { panic!("Got a wrong result accessing a one element data after a discard"); } diff --git a/core/utils/historical-data/src/synch_linear_transaction.rs b/core/utils/historical-data/src/synch_linear_transaction.rs index 939265b227efa..28efe6a9d6705 100644 --- a/core/utils/historical-data/src/synch_linear_transaction.rs +++ b/core/utils/historical-data/src/synch_linear_transaction.rs @@ -27,7 +27,7 @@ //! //! The only global state is a counter of overlayed transaction layer. //! Committing or discarding a layer must use this counter. -//! +//! //! # Local state //! //! Local state is either a committed state (this is a single first independant level @@ -36,10 +36,10 @@ use rstd::vec::Vec; use crate::PruneResult; -/// Global states is a simple counter to the current overlay layer index. +/// Global state is a simple counter to the current overlay layer index. #[derive(Debug, Clone, PartialEq, Eq)] pub struct States(usize); - + impl Default for States { fn default() -> Self { // we default to 1 to be able to discard this transaction. @@ -67,14 +67,14 @@ impl States { /// 0 can be use only to apply transaction change, in /// this case transaction need to be restored to a valid /// state afterward. - pub fn ensure_running(&mut self) { + pub fn finalize_discard(&mut self) { if self.0 == 0 { self.0 = 1; } } /// Discard prospective changes to state. - /// It does not reverts actual values. + /// It does not reverts actual values. /// A subsequent synchronisation of stored values is needed. pub fn discard_prospective(&mut self) { self.0 = 1; @@ -164,7 +164,7 @@ impl States { /// It does not reverts actual values. /// A subsequent synchronisation of stored values is needed. pub fn commit_transaction(&mut self) { - if self.0 > 0 { + if self.0 > 1 { self.0 -= 1; } } @@ -174,25 +174,19 @@ impl States { /// Multiple discard can be applied at the same time. /// Returns true if value is still needed. pub fn apply_commit_transaction(&self, value: &mut History) -> PruneResult { - let target_overlay = if self.0 > 0 { - self.0 - } else { - // `ensure_running` will put state back to one after - 1 - }; let mut new_value = None; for i in (0 .. value.0.len()).rev() { if let HistoricalValue { value: _, index: State::Transaction(ix), } = value.0[i] { - if ix > target_overlay { + if ix > self.0 { if let Some(v) = value.0.pop() { if new_value.is_none() { new_value = Some(v.value); } } - } else if ix == target_overlay && new_value.is_some() { + } else if ix == self.0 && new_value.is_some() { let _ = value.0.pop(); } else { break } } else { break } @@ -200,7 +194,7 @@ impl States { if let Some(new_value) = new_value { value.0.push(HistoricalValue { value: new_value, - index: State::Transaction(target_overlay), + index: State::Transaction(self.0), }); return PruneResult::Changed; } @@ -226,7 +220,6 @@ impl State { } } } - /// An entry at a given history height. pub type HistoricalValue = crate::HistoricalValue; From 55d112ac4de46af2bfa070a29b0141619c96be97 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 Nov 2019 16:49:35 +0100 Subject: [PATCH 082/129] renaming PruneResult to CleaningResult. move back history logic to crate root, and use smallvec on it. remove blank lines. --- core/state-machine/src/overlayed_changes.rs | 21 +++--- core/utils/historical-data/src/lib.rs | 48 ++++++++++++-- .../src/synch_linear_transaction.rs | 65 +++++-------------- 3 files changed, 70 insertions(+), 64 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 508d39b7aca5c..481af6dbf0bfb 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -25,7 +25,7 @@ use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use primitives::storage::well_known_keys::EXTRINSIC_INDEX; use historical_data::synch_linear_transaction::{History, HistoricalValue, States}; -use historical_data::PruneResult; +use historical_data::CleaningResult; /// The overlayed changes to state to be queried on top of the backend. /// @@ -152,9 +152,9 @@ impl OverlayedChangeSet { pub fn discard_prospective(&mut self) { self.states.discard_prospective(); let states = &self.states; - self.top.retain(|_, h_value| states.apply_discard_prospective(h_value) != PruneResult::Cleared); + self.top.retain(|_, h_value| states.apply_discard_prospective(h_value) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_discard_prospective(h_value) != PruneResult::Cleared); + m.retain(|_, h_value| states.apply_discard_prospective(h_value) != CleaningResult::Cleared); !m.is_empty() }); } @@ -163,9 +163,9 @@ impl OverlayedChangeSet { pub fn commit_prospective(&mut self) { self.states.commit_prospective(); let states = &self.states; - self.top.retain(|_, h_value| states.apply_commit_prospective(h_value) != PruneResult::Cleared); + self.top.retain(|_, h_value| states.apply_commit_prospective(h_value) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_commit_prospective(h_value) != PruneResult::Cleared); + m.retain(|_, h_value| states.apply_commit_prospective(h_value) != CleaningResult::Cleared); !m.is_empty() }); } @@ -180,9 +180,9 @@ impl OverlayedChangeSet { pub fn discard_transaction(&mut self) { self.states.discard_transaction(); let states = &self.states; - self.top.retain(|_, h_value| states.apply_discard_transaction(h_value) != PruneResult::Cleared); + self.top.retain(|_, h_value| states.apply_discard_transaction(h_value) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_discard_transaction(h_value) != PruneResult::Cleared); + m.retain(|_, h_value| states.apply_discard_transaction(h_value) != CleaningResult::Cleared); !m.is_empty() }); self.states.finalize_discard(); @@ -192,9 +192,9 @@ impl OverlayedChangeSet { pub fn commit_transaction(&mut self) { self.states.commit_transaction(); let states = &self.states; - self.top.retain(|_, h_value| states.apply_commit_transaction(h_value) != PruneResult::Cleared); + self.top.retain(|_, h_value| states.apply_commit_transaction(h_value) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_commit_transaction(h_value) != PruneResult::Cleared); + m.retain(|_, h_value| states.apply_commit_transaction(h_value) != CleaningResult::Cleared); !m.is_empty() }); } @@ -320,7 +320,6 @@ impl OverlayedChanges { self.changes_trie_config.take() } - /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. @@ -712,7 +711,6 @@ mod tests { Default::default()); } - #[test] fn overlayed_storage_transactions() { let mut overlayed = OverlayedChanges::default(); @@ -738,7 +736,6 @@ mod tests { overlayed.commit_transaction(); assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); - overlayed.discard_transaction(); assert_eq!(overlayed.storage(&key), None); // basic transaction test diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index 72d3864a8d0aa..54e5128e71b3c 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -18,7 +18,6 @@ //! This is use to store historical information for an item, //! and does have to include any proof. - #![cfg_attr(not(feature = "std"), no_std)] pub mod synch_linear_transaction; @@ -47,10 +46,51 @@ impl HistoricalValue { #[derive(PartialEq)] #[cfg_attr(any(test, feature = "test"), derive(Debug))] -/// Prunning result to be able to proceed -/// with further update if the value needs it. -pub enum PruneResult { +/// Results from cleaning a data with history. +/// It should be use to update from the calling context, +/// for instance remove this data from a map if it was cleared. +pub enum CleaningResult { Unchanged, Changed, Cleared, } + +/// History of value and their state. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct History(pub(crate) smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>); + +impl Default for History { + fn default() -> Self { + History(Default::default()) + } +} + +/// Size of preallocated history per element. +/// Currently at two for committed and prospective only. +/// It means that using transaction in a module got a direct allocation cost. +#[cfg(feature = "std")] +const ALLOCATED_HISTORY: usize = 2; + +impl History { + /// Get current number of stored historical values. + pub fn len(&self) -> usize { + self.0.len() + } + + #[cfg(any(test, feature = "test-helpers"))] + /// Create an history from an existing history. + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = History::default(); + for v in input { + history.push_unchecked(v); + } + history + } + + /// Push a value without checking if it can overwrite the current + /// state. + pub fn push_unchecked(&mut self, item: HistoricalValue) { + self.0.push(item); + } +} diff --git a/core/utils/historical-data/src/synch_linear_transaction.rs b/core/utils/historical-data/src/synch_linear_transaction.rs index 28efe6a9d6705..9570021467912 100644 --- a/core/utils/historical-data/src/synch_linear_transaction.rs +++ b/core/utils/historical-data/src/synch_linear_transaction.rs @@ -33,8 +33,7 @@ //! Local state is either a committed state (this is a single first independant level //! of transaction) or a reference to the transaction counter in use in time of creation. -use rstd::vec::Vec; -use crate::PruneResult; +use crate::CleaningResult; /// Global state is a simple counter to the current overlay layer index. #[derive(Debug, Clone, PartialEq, Eq)] @@ -54,7 +53,6 @@ impl States { State::Transaction(self.0) } - /// Build any state for testing only. #[cfg(any(test, feature = "test-helpers"))] pub fn test_state( @@ -81,19 +79,19 @@ impl States { } /// Update a value to a new prospective. - pub fn apply_discard_prospective(&self, value: &mut History) -> PruneResult { + pub fn apply_discard_prospective(&self, value: &mut History) -> CleaningResult { if value.0.len() == 0 { - return PruneResult::Cleared; + return CleaningResult::Cleared; } if value.0[0].index == State::Committed { if value.0.len() == 1 { - return PruneResult::Unchanged; + return CleaningResult::Unchanged; } value.0.truncate(1); } else { value.0.clear(); } - PruneResult::Changed + CleaningResult::Changed } /// Commit prospective changes to state. @@ -104,22 +102,22 @@ impl States { /// Update a value to a new prospective. /// Multiple commit can be applied at the same time. - pub fn apply_commit_prospective(&self, value: &mut History) -> PruneResult { + pub fn apply_commit_prospective(&self, value: &mut History) -> CleaningResult { if value.0.len() == 0 { - return PruneResult::Cleared; + return CleaningResult::Cleared; } if value.0.len() == 1 { if value.0[0].index != State::Committed { value.0[0].index = State::Committed; } else { - return PruneResult::Unchanged; + return CleaningResult::Unchanged; } } else if let Some(mut v) = value.0.pop() { v.index = State::Committed; value.0.clear(); value.0.push(v); } - PruneResult::Changed + CleaningResult::Changed } /// Create a new transactional layer. @@ -139,7 +137,7 @@ impl States { /// Update a value to previous transaction. /// Multiple discard can be applied at the same time. /// Returns true if value is still needed. - pub fn apply_discard_transaction(&self, value: &mut History) -> PruneResult { + pub fn apply_discard_transaction(&self, value: &mut History) -> CleaningResult { let init_len = value.0.len(); for i in (0 .. value.0.len()).rev() { if let HistoricalValue { @@ -152,11 +150,11 @@ impl States { } else { break } } if value.0.len() == 0 { - PruneResult::Cleared + CleaningResult::Cleared } else if value.0.len() != init_len { - PruneResult::Changed + CleaningResult::Changed } else { - PruneResult::Unchanged + CleaningResult::Unchanged } } @@ -173,7 +171,7 @@ impl States { /// after one or more `commit_transaction` calls. /// Multiple discard can be applied at the same time. /// Returns true if value is still needed. - pub fn apply_commit_transaction(&self, value: &mut History) -> PruneResult { + pub fn apply_commit_transaction(&self, value: &mut History) -> CleaningResult { let mut new_value = None; for i in (0 .. value.0.len()).rev() { if let HistoricalValue { @@ -196,9 +194,9 @@ impl States { value: new_value, index: State::Transaction(self.0), }); - return PruneResult::Changed; + return CleaningResult::Changed; } - PruneResult::Unchanged + CleaningResult::Unchanged } } @@ -224,33 +222,10 @@ impl State { pub type HistoricalValue = crate::HistoricalValue; /// History of value and their state. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] -pub struct History(pub(crate) Vec>); - -impl Default for History { - fn default() -> Self { - History(Default::default()) - } -} +pub type History = crate::History; impl History { - /// Get current number of stored historical values. - pub fn len(&self) -> usize { - self.0.len() - } - - #[cfg(any(test, feature = "test-helpers"))] - /// Create an history from an existing history. - pub fn from_iter(input: impl IntoIterator>) -> Self { - let mut history = History::default(); - for v in input { - history.push_unchecked(v); - } - history - } - /// Set a value, it uses a global state as parameter. pub fn set(&mut self, states: &States, value: V) { if let Some(v) = self.0.last_mut() { @@ -269,12 +244,6 @@ impl History { }); } - /// Push a value without checking if it can overwrite the current - /// state. - pub fn push_unchecked(&mut self, item: HistoricalValue) { - self.0.push(item); - } - /// Access to the latest pending value. pub fn get(&self) -> Option<&V> { self.0.last().map(|h| &h.value) From 741d4206e96729935f86522e6ef1af7eca6db900 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 14 Nov 2019 17:22:32 +0100 Subject: [PATCH 083/129] fix compilation issue --- core/utils/historical-data/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index 54e5128e71b3c..3b01ed9ecd9df 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -69,7 +69,6 @@ impl Default for History { /// Size of preallocated history per element. /// Currently at two for committed and prospective only. /// It means that using transaction in a module got a direct allocation cost. -#[cfg(feature = "std")] const ALLOCATED_HISTORY: usize = 2; impl History { From 19b101462307a47f2bd4344758eb7d18b3f152f1 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 15 Nov 2019 10:09:58 +0100 Subject: [PATCH 084/129] Field name for states. Review gramar fixes, trailing ws and doc improvments. --- core/state-machine/src/overlayed_changes.rs | 44 +++++----- core/utils/historical-data/src/lib.rs | 7 +- .../src/synch_linear_transaction.rs | 83 ++++++++++--------- 3 files changed, 64 insertions(+), 70 deletions(-) diff --git a/core/state-machine/src/overlayed_changes.rs b/core/state-machine/src/overlayed_changes.rs index 481af6dbf0bfb..49df7ef1ff2cc 100644 --- a/core/state-machine/src/overlayed_changes.rs +++ b/core/state-machine/src/overlayed_changes.rs @@ -53,14 +53,10 @@ pub struct OverlayedValue { /// Overlayed change set, keep history of values. /// -/// This does not work by stacking hashmaps per transaction, -/// but by storing history of a value instead. -/// Value validity is given by a global indexed state history. -/// When dropping or committing a transaction layer, -/// full history for each values is kept until -/// next garbage collection. -/// Note that on mutable access terminal values corresponding -/// to a deleted layer are also dropped. +/// It stores hashmap containing a linear history of value. +/// Update on commit and discard operation (prospective or +/// overlay), are done in a synchronous manner by updating +/// all keys of the hashmap. #[derive(Debug, Clone, Default)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { @@ -89,15 +85,15 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { /// Variant of `set` value that update extrinsics. /// It does remove latest dropped values. fn set_with_extrinsic_overlayed_value( - h_value: &mut History, + history: &mut History, states: &States, value: Option>, extrinsic_index: Option, ) { if let Some(extrinsic) = extrinsic_index { - set_with_extrinsic_inner_overlayed_value(h_value, states, value, extrinsic) + set_with_extrinsic_inner_overlayed_value(history, states, value, extrinsic) } else { - h_value.set(states, OverlayedValue { + history.set(states, OverlayedValue { value, extrinsics: None, }) @@ -105,13 +101,13 @@ fn set_with_extrinsic_overlayed_value( } fn set_with_extrinsic_inner_overlayed_value( - h_value: &mut History, + history: &mut History, states: &States, value: Option>, extrinsic_index: u32, ) { let state = states.as_state(); - if let Some(current) = h_value.get_mut() { + if let Some(current) = history.get_mut() { if current.index == state { current.value.value = value; current.value.extrinsics.get_or_insert_with(Default::default) @@ -120,7 +116,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.value.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.push_unchecked(HistoricalValue { + history.push_unchecked(HistoricalValue { index: state, value: OverlayedValue { value, @@ -132,7 +128,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics: Option> = None; extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - h_value.push_unchecked(HistoricalValue { + history.push_unchecked(HistoricalValue { index: state, value: OverlayedValue { value, @@ -151,10 +147,9 @@ impl OverlayedChangeSet { /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.states.discard_prospective(); - let states = &self.states; - self.top.retain(|_, h_value| states.apply_discard_prospective(h_value) != CleaningResult::Cleared); + self.top.retain(|_, history| States::apply_discard_prospective(history) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_discard_prospective(h_value) != CleaningResult::Cleared); + m.retain(|_, history| States::apply_discard_prospective(history) != CleaningResult::Cleared); !m.is_empty() }); } @@ -162,10 +157,9 @@ impl OverlayedChangeSet { /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.states.commit_prospective(); - let states = &self.states; - self.top.retain(|_, h_value| states.apply_commit_prospective(h_value) != CleaningResult::Cleared); + self.top.retain(|_, history| States::apply_commit_prospective(history) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_commit_prospective(h_value) != CleaningResult::Cleared); + m.retain(|_, history| States::apply_commit_prospective(history) != CleaningResult::Cleared); !m.is_empty() }); } @@ -180,9 +174,9 @@ impl OverlayedChangeSet { pub fn discard_transaction(&mut self) { self.states.discard_transaction(); let states = &self.states; - self.top.retain(|_, h_value| states.apply_discard_transaction(h_value) != CleaningResult::Cleared); + self.top.retain(|_, history| states.apply_discard_transaction(history) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_discard_transaction(h_value) != CleaningResult::Cleared); + m.retain(|_, history| states.apply_discard_transaction(history) != CleaningResult::Cleared); !m.is_empty() }); self.states.finalize_discard(); @@ -192,9 +186,9 @@ impl OverlayedChangeSet { pub fn commit_transaction(&mut self) { self.states.commit_transaction(); let states = &self.states; - self.top.retain(|_, h_value| states.apply_commit_transaction(h_value) != CleaningResult::Cleared); + self.top.retain(|_, history| states.apply_commit_transaction(history) != CleaningResult::Cleared); self.children.retain(|_, m| { - m.retain(|_, h_value| states.apply_commit_transaction(h_value) != CleaningResult::Cleared); + m.retain(|_, history| states.apply_commit_transaction(history) != CleaningResult::Cleared); !m.is_empty() }); } diff --git a/core/utils/historical-data/src/lib.rs b/core/utils/historical-data/src/lib.rs index 3b01ed9ecd9df..2765531845d74 100644 --- a/core/utils/historical-data/src/lib.rs +++ b/core/utils/historical-data/src/lib.rs @@ -14,9 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Data storage containing multiple state for a value. -//! This is use to store historical information for an item, -//! and does have to include any proof. +//! Data storage containing multiple states for a value. +//! This is used to store historical information for an item. #![cfg_attr(not(feature = "std"), no_std)] @@ -47,7 +46,7 @@ impl HistoricalValue { #[derive(PartialEq)] #[cfg_attr(any(test, feature = "test"), derive(Debug))] /// Results from cleaning a data with history. -/// It should be use to update from the calling context, +/// It should be used to update from the calling context, /// for instance remove this data from a map if it was cleared. pub enum CleaningResult { Unchanged, diff --git a/core/utils/historical-data/src/synch_linear_transaction.rs b/core/utils/historical-data/src/synch_linear_transaction.rs index 9570021467912..2a7be23481faf 100644 --- a/core/utils/historical-data/src/synch_linear_transaction.rs +++ b/core/utils/historical-data/src/synch_linear_transaction.rs @@ -27,7 +27,7 @@ //! //! The only global state is a counter of overlayed transaction layer. //! Committing or discarding a layer must use this counter. -//! +//! //! # Local state //! //! Local state is either a committed state (this is a single first independant level @@ -37,12 +37,12 @@ use crate::CleaningResult; /// Global state is a simple counter to the current overlay layer index. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct States(usize); - +pub struct States { current_layer: usize } + impl Default for States { fn default() -> Self { // we default to 1 to be able to discard this transaction. - States(1) + States { current_layer: 1 } } } @@ -50,15 +50,15 @@ impl States { /// Get corresponding current state. pub fn as_state(&self) -> State { - State::Transaction(self.0) + State::Transaction(self.current_layer) } /// Build any state for testing only. #[cfg(any(test, feature = "test-helpers"))] pub fn test_state( - current_layer_number: usize, + current_layer: usize, ) -> Self { - States(current_layer_number) + States { current_layer } } /// By default the current transaction is at least 1. @@ -66,20 +66,20 @@ impl States { /// this case transaction need to be restored to a valid /// state afterward. pub fn finalize_discard(&mut self) { - if self.0 == 0 { - self.0 = 1; + if self.current_layer == 0 { + self.current_layer = 1; } } /// Discard prospective changes to state. - /// It does not reverts actual values. - /// A subsequent synchronisation of stored values is needed. + /// It does not revert actual values. + /// A subsequent update of all related stored history is needed. pub fn discard_prospective(&mut self) { - self.0 = 1; + self.current_layer = 1; } - /// Update a value to a new prospective. - pub fn apply_discard_prospective(&self, value: &mut History) -> CleaningResult { + /// After a prospective was discarded, clear prospective history. + pub fn apply_discard_prospective(value: &mut History) -> CleaningResult { if value.0.len() == 0 { return CleaningResult::Cleared; } @@ -95,14 +95,13 @@ impl States { } /// Commit prospective changes to state. - /// A subsequent synchronisation of stored values is needed. + /// A subsequent update of all related stored history is needed. pub fn commit_prospective(&mut self) { - self.0 = 1; + self.current_layer = 1; } - /// Update a value to a new prospective. - /// Multiple commit can be applied at the same time. - pub fn apply_commit_prospective(&self, value: &mut History) -> CleaningResult { + /// After a prospective was committed, commit pending value and clear existing history. + pub fn apply_commit_prospective(value: &mut History) -> CleaningResult { if value.0.len() == 0 { return CleaningResult::Cleared; } @@ -122,21 +121,20 @@ impl States { /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.0 += 1; + self.current_layer += 1; } /// Discard a transactional layer. - /// It does not reverts actual values. - /// A subsequent synchronisation of stored values is needed. + /// It does not revert actual values. + /// A subsequent update of all related stored history is needed. pub fn discard_transaction(&mut self) { - if self.0 > 0 { - self.0 -= 1; + if self.current_layer > 0 { + self.current_layer -= 1; } } /// Update a value to previous transaction. /// Multiple discard can be applied at the same time. - /// Returns true if value is still needed. pub fn apply_discard_transaction(&self, value: &mut History) -> CleaningResult { let init_len = value.0.len(); for i in (0 .. value.0.len()).rev() { @@ -144,7 +142,7 @@ impl States { value: _, index: State::Transaction(ix), } = value.0[i] { - if ix > self.0 { + if ix > self.current_layer { let _ = value.0.pop(); } else { break } } else { break } @@ -159,18 +157,17 @@ impl States { } /// Discard a transactional layer. - /// It does not reverts actual values. - /// A subsequent synchronisation of stored values is needed. + /// It does not revert actual values. + /// A subsequent update of all related stored history is needed. pub fn commit_transaction(&mut self) { - if self.0 > 1 { - self.0 -= 1; + if self.current_layer > 1 { + self.current_layer -= 1; } } /// Update a value to be the best historical value /// after one or more `commit_transaction` calls. /// Multiple discard can be applied at the same time. - /// Returns true if value is still needed. pub fn apply_commit_transaction(&self, value: &mut History) -> CleaningResult { let mut new_value = None; for i in (0 .. value.0.len()).rev() { @@ -178,13 +175,13 @@ impl States { value: _, index: State::Transaction(ix), } = value.0[i] { - if ix > self.0 { + if ix > self.current_layer { if let Some(v) = value.0.pop() { if new_value.is_none() { new_value = Some(v.value); } } - } else if ix == self.0 && new_value.is_some() { + } else if ix == self.current_layer && new_value.is_some() { let _ = value.0.pop(); } else { break } } else { break } @@ -192,7 +189,7 @@ impl States { if let Some(new_value) = new_value { value.0.push(HistoricalValue { value: new_value, - index: State::Transaction(self.0), + index: State::Transaction(self.current_layer), }); return CleaningResult::Changed; } @@ -200,12 +197,15 @@ impl States { } } -/// Possible state for a historical value, committed -/// is not touched by transactional action, transaction -/// stored the transaction index of insertion. +/// State for a historical value in a transaction +/// synchronously managed. #[derive(Debug, Clone, PartialEq, Eq)] pub enum State { + /// Committed, transactional action do not + /// change this state. Committed, + /// Value in a transaction, contains the current + /// number of transaction at creation. Transaction(usize), } @@ -229,18 +229,18 @@ impl History { /// Set a value, it uses a global state as parameter. pub fn set(&mut self, states: &States, value: V) { if let Some(v) = self.0.last_mut() { - debug_assert!(v.index.transaction_index().unwrap_or(0) <= states.0, + debug_assert!(v.index.transaction_index().unwrap_or(0) <= states.current_layer, "History expects \ only new values at the latest state, some state has not \ synchronized properly"); - if v.index.transaction_index() == Some(states.0) { + if v.index.transaction_index() == Some(states.current_layer) { v.value = value; return; } } self.0.push(HistoricalValue { value, - index: State::Transaction(states.0), + index: State::Transaction(states.current_layer), }); } @@ -249,7 +249,7 @@ impl History { self.0.last().map(|h| &h.value) } - /// Get latest value, consuming the historical data. + /// Extracts the latest value if there is one. pub fn into_pending(mut self) -> Option { if let Some(v) = self.0.pop() { Some(v.value) @@ -294,6 +294,7 @@ impl History { } } + /// Extracts the committed value if there is one. pub fn into_committed(mut self) -> Option { self.0.truncate(1); if let Some(HistoricalValue { From 4b90775a73b19e0dc370f1af1e86e34f6f371b5f Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 15 Nov 2019 11:19:07 +0100 Subject: [PATCH 085/129] Fix conflicts. --- Cargo.lock | 31 ++--- Cargo.toml | 118 +----------------- core/test-runtime/Cargo.toml | 85 ------------- primitives/state-machine/Cargo.toml | 4 +- .../state-machine/src/overlayed_changes.rs | 21 +--- test/utils/runtime/Cargo.toml | 2 + utils/historical-data/Cargo.toml | 2 +- 7 files changed, 19 insertions(+), 244 deletions(-) delete mode 100644 core/test-runtime/Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index 2e5c9f845b5fb..b86381ce34f6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3826,14 +3826,6 @@ dependencies = [ name = "paint-treasury" version = "2.0.0" dependencies = [ -<<<<<<< HEAD - "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_pcg 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -======= "paint-balances 2.0.0", "paint-support 2.0.0", "paint-system 2.0.0", @@ -3843,7 +3835,6 @@ dependencies = [ "sr-primitives 2.0.0", "sr-std 2.0.0", "substrate-primitives 2.0.0", ->>>>>>> master ] [[package]] @@ -3949,21 +3940,8 @@ version = "0.40.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -<<<<<<< HEAD -name = "rand_pcg" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -======= name = "parity-wasm" version = "0.41.0" ->>>>>>> master source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -4373,6 +4351,7 @@ dependencies = [ "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4480,6 +4459,14 @@ dependencies = [ "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rand_xorshift" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index a672048cc304f..a472450afff73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,121 +1,5 @@ [workspace] members = [ -<<<<<<< HEAD - "core/authority-discovery", - "core/application-crypto", - "core/chain-spec", - "core/chain-spec/derive", - "core/cli", - "core/client", - "core/client/db", - "core/client/header-metadata", - "core/consensus/aura", - "core/consensus/babe", - "core/consensus/common", - "core/consensus/rhd", - "core/consensus/slots", - "core/consensus/uncles", - "core/consensus/pow", - "core/block-builder", - "core/block-builder/runtime-api", - "core/executor", - "core/executor/runtime-test", - "core/externalities", - "core/finality-grandpa", - "core/finality-grandpa/primitives", - "core/inherents", - "core/keyring", - "core/keystore", - "core/network", - "core/panic-handler", - "core/primitives", - "core/rpc", - "core/rpc/primitives", - "core/rpc-servers", - "core/runtime-interface", - "core/runtime-interface/proc-macro", - "core/runtime-interface/test-wasm", - "core/serializer", - "core/service", - "core/service/test", - "core/session", - "core/sr-api", - "core/sr-api/proc-macro", - "core/sr-api/test", - "core/sr-arithmetic", - "core/sr-io", - "core/sr-primitives", - "core/sr-staking-primitives", - "core/sr-sandbox", - "core/sr-std", - "core/sr-version", - "core/state-machine", - "core/telemetry", - "core/test-client", - "core/test-runtime", - "core/test-runtime/client", - "core/transaction-pool", - "core/transaction-pool/graph", - "core/transaction-pool/runtime-api", - "core/trie", - "core/utils/fork-tree", - "core/utils/historical-data", - "core/utils/wasm-builder", - "core/utils/wasm-builder-runner", - "core/wasm-interface", - "srml/support", - "srml/support/rpc", - "srml/support/procedural", - "srml/support/procedural/tools", - "srml/support/procedural/tools/derive", - "srml/support/test", - "srml/authorship", - "srml/assets", - "srml/aura", - "srml/balances", - "srml/contracts", - "srml/contracts/rpc", - "srml/collective", - "srml/democracy", - "srml/elections", - "srml/elections-phragmen", - "srml/example", - "srml/executive", - "srml/finality-tracker", - "srml/generic-asset", - "srml/grandpa", - "srml/im-online", - "srml/authority-discovery", - "srml/indices", - "srml/membership", - "srml/metadata", - "srml/nicks", - "srml/offences", - "srml/randomness-collective-flip", - "srml/scored-pool", - "srml/session", - "srml/staking", - "srml/staking/reward-curve", - "srml/sudo", - "srml/system", - "srml/system/rpc", - "srml/timestamp", - "srml/treasury", - "srml/transaction-payment", - "srml/transaction-payment/rpc", - "srml/utility", - "srml/evm", - "node/cli", - "node/executor", - "node/primitives", - "node/rpc", - "node/rpc-client", - "node/runtime", - "node/testing", - "node-template", - "subkey", - "test-utils/chain-spec-builder", -======= "bin/node-template", "bin/node-template/runtime", "bin/node/cli", @@ -249,8 +133,8 @@ members = [ "test/utils/transaction-factory", "utils/build-script-utils", "utils/fork-tree", + "utils/historical-data", "utils/wasm-builder", ->>>>>>> master ] [profile.release] diff --git a/core/test-runtime/Cargo.toml b/core/test-runtime/Cargo.toml deleted file mode 100644 index c765101842a18..0000000000000 --- a/core/test-runtime/Cargo.toml +++ /dev/null @@ -1,85 +0,0 @@ -[package] -name = "substrate-test-runtime" -version = "2.0.0" -authors = ["Parity Technologies "] -edition = "2018" -build = "build.rs" - -[dependencies] -log = { version = "0.4.8", optional = true } -serde = { version = "1.0.101", optional = true, features = ["derive"] } -codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } -keyring = { package = "substrate-keyring", path = "../keyring", optional = true } -sr-api = { path = "../sr-api", default-features = false } -substrate-client = { path = "../client", optional = true } -primitives = { package = "substrate-primitives", path = "../primitives", default-features = false } -app-crypto = { package = "substrate-application-crypto", path = "../application-crypto", default-features = false } -inherents = { package = "substrate-inherents", path = "../inherents", default-features = false } -aura-primitives = { package = "substrate-consensus-aura-primitives", path = "../consensus/aura/primitives", default-features = false } -babe-primitives = { package = "substrate-consensus-babe-primitives", path = "../consensus/babe/primitives", default-features = false } -rstd = { package = "sr-std", path = "../sr-std", default-features = false } -runtime_io = { package = "sr-io", path = "../sr-io", default-features = false } -sr-primitives = { path = "../sr-primitives", default-features = false } -session = { package = "substrate-session", path = "../session", default-features = false } -runtime_version = { package = "sr-version", path = "../sr-version", default-features = false } -runtime_support = { package = "srml-support", path = "../../srml/support", default-features = false } -substrate-trie = { path = "../trie", default-features = false } -trie-db = { version = "0.15.2", default-features = false } -memory-db = { version = "0.15.2", default-features = false } -offchain-primitives = { package = "substrate-offchain-primitives", path = "../offchain/primitives", default-features = false} -runtime-interface = { package = "substrate-runtime-interface", path = "../runtime-interface", default-features = false} -executive = { package = "srml-executive", path = "../../srml/executive", default-features = false } -cfg-if = "0.1.10" -srml-babe = { path = "../../srml/babe", default-features = false } -srml-timestamp = { path = "../../srml/timestamp", default-features = false } -srml-system = { path = "../../srml/system", default-features = false } -srml-system-rpc-runtime-api = { path = "../../srml/system/rpc/runtime-api", default-features = false } -historical-data = { path = "../utils/historical-data", default-features = false } -transaction-pool-api = { package = "substrate-transaction-pool-runtime-api", path = "../transaction-pool/runtime-api", default-features = false } -block-builder-api = { package = "substrate-block-builder-runtime-api", path = "../block-builder/runtime-api", default-features = false } - -[dev-dependencies] -substrate-executor = { path = "../executor" } -substrate-test-runtime-client = { path = "./client" } -state_machine = { package = "substrate-state-machine", path = "../state-machine" } - -[build-dependencies] -wasm-builder-runner = { package = "substrate-wasm-builder-runner", version = "1.0.4", path = "../utils/wasm-builder-runner" } - -[features] -default = [ - "std", -] -std = [ - "log", - "serde", - "sr-api/std", - "keyring", - "codec/std", - "rstd/std", - "runtime_io/std", - "runtime_support/std", - "primitives/std", - "inherents/std", - "sr-primitives/std", - "runtime_version/std", - "aura-primitives/std", - "babe-primitives/std", - "primitives/std", - "substrate-trie/std", - "trie-db/std", - "memory-db/std", - "offchain-primitives/std", - "executive/std", - "srml-babe/std", - "srml-timestamp/std", - "srml-system/std", - "srml-system-rpc-runtime-api/std", - "app-crypto/std", - "session/std", - "historical-data/std", - "runtime-interface/std", - "transaction-pool-api/std", - "block-builder-api/std", - "substrate-client", -] diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 9e896129e2087..0835a82a6995b 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -15,7 +15,7 @@ trie = { package = "substrate-trie", path = "../trie" } primitives = { package = "substrate-primitives", path = "../core" } panic-handler = { package = "substrate-panic-handler", path = "../panic-handler" } codec = { package = "parity-scale-codec", version = "1.0.0" } -historical-data = { path = "../utils/historical-data" } +historical-data = { path = "../../utils/historical-data" } num-traits = "0.2.8" rand = "0.7.2" externalities = { package = "substrate-externalities", path = "../externalities" } @@ -24,7 +24,7 @@ externalities = { package = "substrate-externalities", path = "../externalities" hex-literal = "0.2.1" criterion = "0.2" rand = { version = "0.7.1", features = ["small_rng"] } -historical-data = { path = "../utils/historical-data", features = [ "test-helpers" ] } +historical-data = { path = "../../utils/historical-data", features = [ "test-helpers" ] } [[bench]] name = "bench" diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 9df6bd4155aca..2a0251bc631eb 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -537,18 +537,10 @@ mod tests { use crate::ext::Ext; use super::*; -<<<<<<< HEAD:core/state-machine/src/overlayed_changes.rs - fn strip_extrinsic_index(mut map: HashMap, OverlayedValue>) -> HashMap, OverlayedValue> { + fn strip_extrinsic_index(mut map: HashMap, OverlayedValue>) + -> HashMap, OverlayedValue> { map.remove(&EXTRINSIC_INDEX.to_vec()); map -======= - fn strip_extrinsic_index(map: &HashMap, OverlayedValue>) - -> HashMap, OverlayedValue> - { - let mut clone = map.clone(); - clone.remove(&EXTRINSIC_INDEX.to_vec()); - clone ->>>>>>> master:primitives/state-machine/src/overlayed_changes.rs } #[test] @@ -704,15 +696,10 @@ mod tests { assert_eq!(strip_extrinsic_index(overlay.changes.top_prospective()), vec![ -<<<<<<< HEAD:core/state-machine/src/overlayed_changes.rs - (vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }), - (vec![3], OverlayedValue { value: Some(vec![7]), extrinsics: Some(vec![1, 3].into_iter().collect()) }), -======= (vec![1], OverlayedValue { value: Some(vec![8]), - extrinsics: Some(vec![4].into_iter().collect()) }), + extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }), (vec![3], OverlayedValue { value: Some(vec![7]), - extrinsics: Some(vec![3].into_iter().collect()) }), ->>>>>>> master:primitives/state-machine/src/overlayed_changes.rs + extrinsics: Some(vec![1, 3].into_iter().collect()) }), ].into_iter().collect()); overlay.commit_prospective(); diff --git a/test/utils/runtime/Cargo.toml b/test/utils/runtime/Cargo.toml index c405451fa8602..b74439cb93bf3 100644 --- a/test/utils/runtime/Cargo.toml +++ b/test/utils/runtime/Cargo.toml @@ -36,6 +36,7 @@ substrate-client = { path = "../../../client", optional = true } substrate-trie = { path = "../../../primitives/trie", default-features = false } transaction-pool-api = { package = "substrate-transaction-pool-runtime-api", path = "../../../primitives/transaction-pool/runtime-api", default-features = false } trie-db = { version = "0.15.2", default-features = false } +historical-data = { path = "../../../utils/historical-data", default-features = false } [dev-dependencies] substrate-executor = { path = "../../../client/executor" } @@ -80,4 +81,5 @@ std = [ "substrate-trie/std", "transaction-pool-api/std", "trie-db/std", + "historical-data/std", ] diff --git a/utils/historical-data/Cargo.toml b/utils/historical-data/Cargo.toml index 91103903084d7..9cdf00e4e9518 100644 --- a/utils/historical-data/Cargo.toml +++ b/utils/historical-data/Cargo.toml @@ -6,7 +6,7 @@ description = "Data associated with its history" edition = "2018" [dependencies] -rstd = { package = "sr-std", path = "../../sr-std", default-features = false } +rstd = { package = "sr-std", path = "../../primitives/sr-std", default-features = false } smallvec = { version = "0.6", default-features = false } [features] From a81fb12369450a139cd25ac2fa414011030e393c Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 15 Nov 2019 11:58:19 +0100 Subject: [PATCH 086/129] fix ci for checking fuzz --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5730a80e21c0f..07afa7788abb9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -108,7 +108,7 @@ cargo-check-benches: <<: *docker-env script: - BUILD_DUMMY_WASM_BINARY=1 time cargo +nightly check --benches --all - - cd ./core/state-machine/fuzz + - cd ./primitives/state-machine/fuzz - time cargo check - sccache -s From d9ff813bb10dfbb37220c4c3591f48c067960088 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 19 Nov 2019 19:37:17 +0100 Subject: [PATCH 087/129] No sense in having new ext in deprecated interface. --- client/executor/src/deprecated_host_interface.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/client/executor/src/deprecated_host_interface.rs b/client/executor/src/deprecated_host_interface.rs index 33a2dc46ff012..223b13367aa87 100644 --- a/client/executor/src/deprecated_host_interface.rs +++ b/client/executor/src/deprecated_host_interface.rs @@ -1062,18 +1062,6 @@ impl_wasm_host_interface! { } }) } - - ext_storage_start_transaction() { - Ok(runtime_io::storage::start_transaction()) - } - - ext_storage_discard_transaction() { - Ok(runtime_io::storage::discard_transaction()) - } - - ext_storage_commit_transaction() { - Ok(runtime_io::storage::commit_transaction()) - } } } From fa0ec931d752bf04d6c6a135d43562115423839b Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 19 Nov 2019 21:14:27 +0100 Subject: [PATCH 088/129] Ensure discard prospective. --- primitives/state-machine/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 0d9a008334ba7..4d90402db8981 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -360,6 +360,7 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where if !was_native || result.is_ok() { (result, storage_delta, changes_delta) } else { + self.overlay.discard_prospective(); let (wasm_result, _, wasm_storage_delta, wasm_changes_delta) = self.execute_aux( compute_tx, false, From 5597bf375224b3eadd355b312a1402cac46ea36b Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 2 Dec 2019 14:37:36 +0100 Subject: [PATCH 089/129] update cargo.lock --- Cargo.lock | 3022 ++++++++++++++++++++++++++-------------------------- 1 file changed, 1511 insertions(+), 1511 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bd145b50b2f24..3adb1d3854f88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -476,9 +476,9 @@ dependencies = [ "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", "node-cli 2.0.0", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-keystore 2.0.0", + "sp-core 2.0.0", "structopt 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-keystore 2.0.0", - "substrate-primitives 2.0.0", ] [[package]] @@ -1246,10 +1246,10 @@ dependencies = [ "pallet-transaction-payment 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -1258,8 +1258,8 @@ version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -1277,13 +1277,13 @@ dependencies = [ "paste 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-arithmetic 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", + "sp-arithmetic 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "sp-std 2.0.0", "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1325,11 +1325,11 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", "trybuild 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1343,11 +1343,11 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", + "sp-version 2.0.0", ] [[package]] @@ -1355,7 +1355,7 @@ name = "frame-system-rpc-runtime-api" version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", + "sp-api 2.0.0", ] [[package]] @@ -1367,10 +1367,10 @@ dependencies = [ "pallet-balances 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -1809,7 +1809,7 @@ name = "historical-data" version = "2.0.0" dependencies = [ "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -2998,38 +2998,38 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-authority-discovery 2.0.0", + "sc-basic-authority 2.0.0", + "sc-chain-spec 2.0.0", + "sc-cli 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-client-db 2.0.0", + "sc-consensus-babe 2.0.0", + "sc-finality-grandpa 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-offchain 2.0.0", + "sc-rpc 2.0.0", + "sc-service 2.0.0", + "sc-service-test 2.0.0", + "sc-telemetry 2.0.0", "sc-transaction-pool 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-authority-discovery 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-finality-granpda 2.0.0", "sp-finality-tracker 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", "sp-timestamp 2.0.0", "sp-transaction-pool-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", "structopt 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-authority-discovery 2.0.0", - "substrate-authority-discovery-primitives 2.0.0", - "substrate-basic-authorship 2.0.0", "substrate-build-script-utils 2.0.0", - "substrate-chain-spec 2.0.0", - "substrate-cli 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-client-db 2.0.0", - "substrate-consensus-babe 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-finality-grandpa 2.0.0", - "substrate-finality-grandpa-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-offchain 2.0.0", - "substrate-primitives 2.0.0", - "substrate-rpc 2.0.0", - "substrate-service 2.0.0", - "substrate-service-test 2.0.0", - "substrate-telemetry 2.0.0", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "transaction-factory 0.0.1", @@ -3057,13 +3057,13 @@ dependencies = [ "pallet-transaction-payment 2.0.0", "pallet-treasury 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "substrate-executor 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", + "sc-executor 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "sp-trie 2.0.0", "substrate-test-client 2.0.0", - "substrate-trie 2.0.0", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3073,9 +3073,9 @@ name = "node-primitives" version = "2.0.0" dependencies = [ "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "substrate-primitives 2.0.0", - "substrate-serializer 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "sp-serializer 2.0.0", ] [[package]] @@ -3087,9 +3087,9 @@ dependencies = [ "node-runtime 2.0.0", "pallet-contracts-rpc 2.0.0", "pallet-transaction-payment-rpc 2.0.0", + "sc-client 2.0.0", + "sp-runtime 2.0.0", "sp-transaction-pool-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-client 2.0.0", "substrate-frame-rpc-system 2.0.0", ] @@ -3103,7 +3103,7 @@ dependencies = [ "jsonrpc-core-client 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "node-primitives 2.0.0", - "substrate-rpc 2.0.0", + "sc-rpc 2.0.0", ] [[package]] @@ -3146,21 +3146,21 @@ dependencies = [ "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-authority-discovery 2.0.0", + "sp-block-builder 2.0.0", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-offchain 2.0.0", + "sp-runtime 2.0.0", + "sp-sesssion 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", "sp-transaction-pool-runtime-api 2.0.0", - "sr-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-authority-discovery-primitives 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-offchain-primitives 2.0.0", - "substrate-primitives 2.0.0", - "substrate-session 2.0.0", + "sp-version 2.0.0", "substrate-wasm-builder-runner 1.0.4", ] @@ -3175,24 +3175,24 @@ dependencies = [ "node-template-runtime 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-basic-authority 2.0.0", + "sc-cli 2.0.0", + "sc-client 2.0.0", + "sc-consensus-aura 2.0.0", + "sc-executor 2.0.0", + "sc-finality-grandpa 2.0.0", + "sc-network 2.0.0", + "sc-service 2.0.0", "sc-transaction-pool 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-aura 2.0.0", + "sp-core 2.0.0", + "sp-finality-granpda 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", "sp-transaction-pool-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "substrate-basic-authorship 2.0.0", "substrate-build-script-utils 2.0.0", - "substrate-cli 2.0.0", - "substrate-client 2.0.0", - "substrate-consensus-aura 2.0.0", - "substrate-consensus-aura-primitives 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-executor 2.0.0", - "substrate-finality-grandpa 2.0.0", - "substrate-finality-grandpa-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-network 2.0.0", - "substrate-primitives 2.0.0", - "substrate-service 2.0.0", "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "vergen 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3216,18 +3216,18 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-block-builder 2.0.0", + "sp-consensus-aura 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-offchain 2.0.0", + "sp-runtime 2.0.0", + "sp-sesssion 2.0.0", + "sp-std 2.0.0", "sp-transaction-pool-runtime-api 2.0.0", - "sr-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-consensus-aura-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-offchain-primitives 2.0.0", - "substrate-primitives 2.0.0", - "substrate-session 2.0.0", + "sp-version 2.0.0", "substrate-wasm-builder-runner 1.0.4", ] @@ -3250,12 +3250,12 @@ dependencies = [ "pallet-transaction-payment 2.0.0", "pallet-treasury 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "substrate-client 2.0.0", - "substrate-executor 2.0.0", - "substrate-keyring 2.0.0", - "substrate-primitives 2.0.0", + "sc-client 2.0.0", + "sc-executor 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", "substrate-test-client 2.0.0", "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3412,10 +3412,10 @@ dependencies = [ "frame-system 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3429,15 +3429,15 @@ dependencies = [ "pallet-timestamp 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-consensus-aura 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", "sp-timestamp 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-consensus-aura-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", ] [[package]] @@ -3448,14 +3448,14 @@ dependencies = [ "frame-system 2.0.0", "pallet-session 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-authority-discovery-primitives 2.0.0", - "substrate-primitives 2.0.0", + "sp-authority-discovery 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3467,11 +3467,11 @@ dependencies = [ "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "sp-authorship 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3487,15 +3487,15 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", "sp-timestamp 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", + "sp-version 2.0.0", "substrate-test-runtime 2.0.0", ] @@ -3509,10 +3509,10 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3526,10 +3526,10 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3547,11 +3547,11 @@ dependencies = [ "parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)", "pwasm-utils 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-sandbox 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-sandbox 2.0.0", + "sp-std 2.0.0", "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", "wasmi-validation 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3567,9 +3567,9 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "sp-blockchain 2.0.0", - "sr-primitives 2.0.0", - "substrate-primitives 2.0.0", - "substrate-rpc-primitives 2.0.0", + "sp-core 2.0.0", + "sp-rpc 2.0.0", + "sp-runtime 2.0.0", ] [[package]] @@ -3578,9 +3578,9 @@ version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", + "sp-api 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3593,10 +3593,10 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3610,10 +3610,10 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3626,11 +3626,11 @@ dependencies = [ "pallet-balances 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-phragmen 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-phragmen 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", "substrate-test-utils 2.0.0", ] @@ -3648,10 +3648,10 @@ dependencies = [ "rlp 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "sha3 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3663,10 +3663,10 @@ dependencies = [ "pallet-balances 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3678,12 +3678,12 @@ dependencies = [ "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", "sp-finality-tracker 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3694,10 +3694,10 @@ dependencies = [ "frame-system 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3710,12 +3710,12 @@ dependencies = [ "pallet-session 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-finality-grandpa-primitives 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-finality-granpda 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3727,13 +3727,13 @@ dependencies = [ "pallet-authorship 0.1.0", "pallet-session 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3746,11 +3746,11 @@ dependencies = [ "ref_thread_local 0.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-keyring 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3761,10 +3761,10 @@ dependencies = [ "frame-system 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3776,10 +3776,10 @@ dependencies = [ "pallet-balances 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3791,11 +3791,11 @@ dependencies = [ "pallet-balances 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3806,10 +3806,10 @@ dependencies = [ "frame-system 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3821,10 +3821,10 @@ dependencies = [ "pallet-balances 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3838,14 +3838,14 @@ dependencies = [ "pallet-timestamp 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-primitives 2.0.0", - "substrate-trie 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", + "sp-trie 2.0.0", ] [[package]] @@ -3862,13 +3862,13 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "safe-mix 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-staking-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-keyring 2.0.0", - "substrate-phragmen 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-phragmen 2.0.0", + "sp-runtime 2.0.0", + "sp-staking 2.0.0", + "sp-std 2.0.0", "substrate-test-utils 2.0.0", ] @@ -3879,7 +3879,7 @@ dependencies = [ "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", + "sp-runtime 2.0.0", "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3891,10 +3891,10 @@ dependencies = [ "frame-system 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3906,12 +3906,12 @@ dependencies = [ "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", "sp-timestamp 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", ] [[package]] @@ -3923,10 +3923,10 @@ dependencies = [ "pallet-balances 2.0.0", "pallet-transaction-payment-rpc-runtime-api 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3940,9 +3940,9 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "sp-blockchain 2.0.0", - "sr-primitives 2.0.0", - "substrate-primitives 2.0.0", - "substrate-rpc-primitives 2.0.0", + "sp-core 2.0.0", + "sp-rpc 2.0.0", + "sp-runtime 2.0.0", ] [[package]] @@ -3953,9 +3953,9 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", + "sp-api 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -3967,10 +3967,10 @@ dependencies = [ "pallet-balances 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] @@ -4817,6 +4817,606 @@ dependencies = [ "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "sc-application-crypto" +version = "2.0.0" +dependencies = [ + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", + "substrate-test-runtime-client 2.0.0", +] + +[[package]] +name = "sc-authority-discovery" +version = "2.0.0" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "prost 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "prost-build 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-peerset 2.0.0", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-authority-discovery 2.0.0", + "sp-blockchain 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "substrate-test-runtime-client 2.0.0", +] + +[[package]] +name = "sc-basic-authority" +version = "2.0.0" +dependencies = [ + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-block-builder 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-telemetry 2.0.0", + "sc-transaction-pool 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "sp-transaction-pool-api 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-block-builder" +version = "2.0.0" +dependencies = [ + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-block-builder 2.0.0", + "sp-blockchain 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", +] + +[[package]] +name = "sc-chain-spec" +version = "2.0.0" +dependencies = [ + "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-chain-spec-derive 2.0.0", + "sc-network 2.0.0", + "sc-telemetry 2.0.0", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-runtime 2.0.0", +] + +[[package]] +name = "sc-chain-spec-derive" +version = "2.0.0" +dependencies = [ + "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-cli" +version = "2.0.0" +dependencies = [ + "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures01 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "names 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rpassword 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", + "sc-network 2.0.0", + "sc-service 2.0.0", + "sc-telemetry 2.0.0", + "sc-tracing 2.0.0", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-blockchain 2.0.0", + "sp-core 2.0.0", + "sp-keyring 2.0.0", + "sp-panic-handler 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "structopt 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-client" +version = "2.0.0" +dependencies = [ + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-memorydb 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-block-builder 2.0.0", + "sc-client-api 2.0.0", + "sc-client-db 2.0.0", + "sc-executor 2.0.0", + "sc-telemetry 2.0.0", + "sp-api 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-externalities 2.0.0", + "sp-inherents 2.0.0", + "sp-keyring 2.0.0", + "sp-panic-handler 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "sp-std 2.0.0", + "sp-trie 2.0.0", + "sp-version 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-client-api" +version = "2.0.0" +dependencies = [ + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-memorydb 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-block-builder 2.0.0", + "sc-client-db 2.0.0", + "sc-executor 2.0.0", + "sc-telemetry 2.0.0", + "sp-api 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-externalities 2.0.0", + "sp-inherents 2.0.0", + "sp-keyring 2.0.0", + "sp-panic-handler 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "sp-std 2.0.0", + "sp-transaction-pool-api 2.0.0", + "sp-trie 2.0.0", + "sp-version 2.0.0", + "substrate-test-primitives 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-client-db" +version = "2.0.0" +dependencies = [ + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-memorydb 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "kvdb-rocksdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quickcheck 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-executor 2.0.0", + "sc-state-db 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "sp-trie 2.0.0", + "substrate-test-runtime-client 2.0.0", +] + +[[package]] +name = "sc-consensus-aura" +version = "2.0.0" +dependencies = [ + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-consensus-slots 2.0.0", + "sc-executor 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-service 2.0.0", + "sc-telemetry 2.0.0", + "sp-api 2.0.0", + "sp-block-builder 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-aura 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", + "sp-timestamp 2.0.0", + "sp-version 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-consensus-babe" +version = "2.0.0" +dependencies = [ + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fork-tree 2.0.0", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", + "sc-block-builder 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-consensus-slots 2.0.0", + "sc-consensus-uncles 2.0.0", + "sc-executor 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-service 2.0.0", + "sc-telemetry 2.0.0", + "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-block-builder 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", + "sp-timestamp 2.0.0", + "sp-version 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-consensus-pow" +version = "2.0.0" +dependencies = [ + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", + "sp-block-builder 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-pow 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "sp-timestamp 2.0.0", +] + +[[package]] +name = "sc-consensus-slots" +version = "2.0.0" +dependencies = [ + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", + "sc-telemetry 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "substrate-test-runtime-client 2.0.0", +] + +[[package]] +name = "sc-consensus-uncles" +version = "2.0.0" +dependencies = [ + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", + "sp-authorship 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", +] + +[[package]] +name = "sc-executor" +version = "2.0.0" +dependencies = [ + "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cranelift-codegen 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cranelift-entity 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cranelift-frontend 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cranelift-native 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cranelift-wasm 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", + "sc-offchain 2.0.0", + "sc-runtime-test 2.0.0", + "sp-core 2.0.0", + "sp-externalities 2.0.0", + "sp-io 2.0.0", + "sp-panic-handler 2.0.0", + "sp-runtime-interface 2.0.0", + "sp-serializer 2.0.0", + "sp-state-machine 2.0.0", + "sp-trie 2.0.0", + "sp-version 2.0.0", + "sp-wasm-interface 2.0.0", + "substrate-test-runtime 2.0.0", + "test-case 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmtime-environ 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmtime-jit 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmtime-runtime 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-finality-grandpa" +version = "2.0.0" +dependencies = [ + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "finality-grandpa 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fork-tree 2.0.0", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-telemetry 2.0.0", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-finality-granpda 2.0.0", + "sp-finality-tracker 2.0.0", + "sp-inherents 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-keystore" +version = "2.0.0" +dependencies = [ + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-network" +version = "2.0.0" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "erased-serde 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "fork-tree 2.0.0", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "linked_hash_set 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lru 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "quickcheck 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-block-builder 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-peerset 2.0.0", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "slog_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-arithmetic 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", + "substrate-test-client 2.0.0", + "substrate-test-runtime 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "unsigned-varint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "zeroize 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-offchain" +version = "2.0.0" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-rustls 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", + "sc-client-db 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-transaction-pool 2.0.0", + "sp-api 2.0.0", + "sp-core 2.0.0", + "sp-offchain 2.0.0", + "sp-runtime 2.0.0", + "sp-transaction-pool-api 2.0.0", + "substrate-test-runtime-client 2.0.0", + "threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-peerset" +version = "2.0.0" +dependencies = [ + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-rpc" +version = "2.0.0" +dependencies = [ + "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-pubsub 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-executor 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-rpc-api 2.0.0", + "sc-transaction-pool 2.0.0", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-blockchain 2.0.0", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-rpc 2.0.0", + "sp-runtime 2.0.0", + "sp-sesssion 2.0.0", + "sp-state-machine 2.0.0", + "sp-transaction-pool-api 2.0.0", + "sp-version 2.0.0", + "substrate-test-runtime-client 2.0.0", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sc-rpc-api" version = "2.0.0" @@ -4832,10 +5432,156 @@ dependencies = [ "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-transaction-pool-api 2.0.0", - "sr-version 2.0.0", - "substrate-primitives 2.0.0", - "substrate-rpc-primitives 2.0.0", + "sp-core 2.0.0", + "sp-rpc 2.0.0", + "sp-transaction-pool-api 2.0.0", + "sp-version 2.0.0", +] + +[[package]] +name = "sc-rpc-server" +version = "2.0.0" +dependencies = [ + "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-http-server 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-pubsub 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-ws-server 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-runtime 2.0.0", +] + +[[package]] +name = "sc-runtime-test" +version = "2.0.0" +dependencies = [ + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-sandbox 2.0.0", + "sp-std 2.0.0", + "substrate-wasm-builder-runner 1.0.4", +] + +[[package]] +name = "sc-service" +version = "2.0.0" +dependencies = [ + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "grafana-data-source 2.0.0", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-multiaddr 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", + "sc-chain-spec 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-client-db 2.0.0", + "sc-executor 2.0.0", + "sc-finality-grandpa 2.0.0", + "sc-keystore 2.0.0", + "sc-network 2.0.0", + "sc-offchain 2.0.0", + "sc-rpc 2.0.0", + "sc-rpc-server 2.0.0", + "sc-telemetry 2.0.0", + "sc-tracing 2.0.0", + "sc-transaction-pool 2.0.0", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-blockchain 2.0.0", + "sp-consensus 2.0.0", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-finality-granpda 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-sesssion 2.0.0", + "sp-transaction-pool-api 2.0.0", + "sp-transaction-pool-runtime-api 2.0.0", + "substrate-test-runtime-client 2.0.0", + "sysinfo 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", + "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-service-test" +version = "2.0.0" +dependencies = [ + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client 2.0.0", + "sc-network 2.0.0", + "sc-service 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "sp-transaction-pool-api 2.0.0", + "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-state-db" +version = "2.0.0" +dependencies = [ + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", +] + +[[package]] +name = "sc-telemetry" +version = "2.0.0" +dependencies = [ + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "slog-async 2.3.0 (git+https://github.com/paritytech/slog-async)", + "slog-json 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sc-tracing" +version = "2.0.0" +dependencies = [ + "erased-serde 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "grafana-data-source 2.0.0", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-telemetry 2.0.0", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4851,9 +5597,9 @@ dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-runtime 2.0.0", "sp-transaction-pool-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-primitives 2.0.0", "substrate-test-runtime 2.0.0", ] @@ -4866,15 +5612,15 @@ dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client-api 2.0.0", "sc-transaction-graph 2.0.0", + "sp-api 2.0.0", "sp-blockchain 2.0.0", + "sp-core 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", "sp-transaction-pool-api 2.0.0", "sp-transaction-pool-runtime-api 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-client-api 2.0.0", - "substrate-keyring 2.0.0", - "substrate-primitives 2.0.0", "substrate-test-runtime-client 2.0.0", ] @@ -5087,1482 +5833,777 @@ source = "git+https://github.com/paritytech/slog-async#107848e7ded5e80dc43f6296c dependencies = [ "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "slog-json" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", - "erased-serde 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "slog-scope" -version = "4.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arc-swap 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "slog_derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "smallvec" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "smallvec" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "snow" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "soketto" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "http 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)", - "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sourcefile" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "sp-authorship" -version = "2.0.0" -dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", -] - -[[package]] -name = "sp-blockchain" -version = "2.0.0" -dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lru 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-state-machine 2.0.0", -] - -[[package]] -name = "sp-finality-tracker" -version = "2.0.0" -dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", -] - -[[package]] -name = "sp-timestamp" -version = "2.0.0" -dependencies = [ - "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", -] - -[[package]] -name = "sp-transaction-pool-api" -version = "2.0.0" -dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "substrate-primitives 2.0.0", -] - -[[package]] -name = "sp-transaction-pool-runtime-api" -version = "2.0.0" -dependencies = [ - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-primitives 2.0.0", -] - -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "sr-api" -version = "2.0.0" -dependencies = [ - "criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api-proc-macro 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-test-runtime-client 2.0.0", -] - -[[package]] -name = "sr-api-proc-macro" -version = "2.0.0" -dependencies = [ - "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-version 2.0.0", - "substrate-test-runtime-client 2.0.0", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sr-api-test" -version = "2.0.0" -dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rustversion 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-version 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-test-runtime-client 2.0.0", - "trybuild 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sr-arithmetic" -version = "2.0.0" -dependencies = [ - "criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "integer-sqrt 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-debug-derive 2.0.0", -] - -[[package]] -name = "sr-io" -version = "2.0.0" -dependencies = [ - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-externalities 2.0.0", - "substrate-primitives 2.0.0", - "substrate-runtime-interface 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-trie 2.0.0", -] - -[[package]] -name = "sr-primitives" -version = "2.0.0" -dependencies = [ - "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "paste 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-arithmetic 2.0.0", - "sr-io 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", -] - -[[package]] -name = "sr-sandbox" -version = "2.0.0" -dependencies = [ - "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", - "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "sr-staking-primitives" -version = "2.0.0" -dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "sr-std 2.0.0", -] - -[[package]] -name = "sr-std" -version = "2.0.0" - -[[package]] -name = "sr-version" -version = "2.0.0" -dependencies = [ - "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "sr-std 2.0.0", -] - -[[package]] -name = "stable_deref_trait" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "stream-cipher" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "string" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "string-interner" -version = "0.7.1" +name = "slog-json" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "chrono 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "erased-serde 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "strsim" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "structopt" -version = "0.3.4" +name = "slog-scope" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt-derive 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "arc-swap 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "structopt-derive" -version = "0.3.4" +name = "slog_derive" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-error 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "strum" -version = "0.16.0" +name = "smallvec" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "strum_macros 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "strum_macros" -version = "0.16.0" +name = "smallvec" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "subkey" -version = "2.0.0" -dependencies = [ - "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "frame-system 2.0.0", - "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "node-primitives 2.0.0", - "node-runtime 2.0.0", - "pallet-balances 2.0.0", - "pallet-transaction-payment 2.0.0", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-primitives 2.0.0", - "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] -name = "substrate-application-crypto" -version = "2.0.0" +name = "snow" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", - "substrate-test-runtime-client 2.0.0", + "arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-authority-discovery" -version = "2.0.0" +name = "soketto" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "prost 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "prost-build 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-authority-discovery-primitives 2.0.0", - "substrate-client-api 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-peerset 2.0.0", - "substrate-primitives 2.0.0", - "substrate-test-runtime-client 2.0.0", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-codec 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-authority-discovery-primitives" -version = "2.0.0" -dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", -] +name = "sourcefile" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "substrate-basic-authorship" +name = "sp-api" version = "2.0.0" dependencies = [ - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sc-transaction-pool 2.0.0", - "sp-blockchain 2.0.0", - "sp-transaction-pool-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-block-builder 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", - "substrate-telemetry 2.0.0", + "sp-api-proc-macro 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "sp-std 2.0.0", + "sp-version 2.0.0", "substrate-test-runtime-client 2.0.0", - "tokio-executor 0.2.0-alpha.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-bip39" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" +name = "sp-api-proc-macro" +version = "2.0.0" dependencies = [ - "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-runtime 2.0.0", + "sp-version 2.0.0", + "substrate-test-runtime-client 2.0.0", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-block-builder" +name = "sp-api-test" version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rustversion 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", "sp-blockchain 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", + "sp-consensus 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", + "sp-version 2.0.0", + "substrate-test-runtime-client 2.0.0", + "trybuild 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-block-builder-runtime-api" +name = "sp-arithmetic" version = "2.0.0" dependencies = [ + "criterion 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "integer-sqrt 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-inherents 2.0.0", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-debug-derive 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-build-script-utils" -version = "2.0.0" - -[[package]] -name = "substrate-chain-spec" +name = "sp-authority-discovery" version = "2.0.0" dependencies = [ - "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "substrate-chain-spec-derive 2.0.0", - "substrate-network 2.0.0", - "substrate-primitives 2.0.0", - "substrate-telemetry 2.0.0", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", + "sp-api 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-chain-spec-derive" +name = "sp-authorship" version = "2.0.0" dependencies = [ - "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-inherents 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-cli" +name = "sp-block-builder" version = "2.0.0" dependencies = [ - "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", - "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures01 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "names 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rpassword 4.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sr-primitives 2.0.0", - "structopt 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-client-api 2.0.0", - "substrate-keyring 2.0.0", - "substrate-network 2.0.0", - "substrate-panic-handler 2.0.0", - "substrate-primitives 2.0.0", - "substrate-service 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-tracing 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-client" +name = "sp-blockchain" version = "2.0.0" dependencies = [ "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-memorydb 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "lru 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-block-builder 2.0.0", - "substrate-client-api 2.0.0", - "substrate-client-db 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-executor 2.0.0", - "substrate-externalities 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-panic-handler 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-test-runtime-client 2.0.0", - "substrate-trie 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-block-builder 2.0.0", + "sp-consensus 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", ] [[package]] -name = "substrate-client-api" +name = "sp-consensus" version = "2.0.0" dependencies = [ "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-memorydb 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sp-transaction-pool-api 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-block-builder 2.0.0", - "substrate-client-db 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-executor 2.0.0", - "substrate-externalities 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-panic-handler 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-test-primitives 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", + "sp-version 2.0.0", "substrate-test-runtime-client 2.0.0", - "substrate-trie 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-client-db" +name = "sp-consensus-aura" version = "2.0.0" dependencies = [ - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-memorydb 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "kvdb-rocksdb 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "quickcheck 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sr-primitives 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-executor 2.0.0", - "substrate-keyring 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-db 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-test-runtime-client 2.0.0", - "substrate-trie 2.0.0", + "sc-application-crypto 2.0.0", + "sp-api 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", + "sp-timestamp 2.0.0", ] [[package]] -name = "substrate-consensus-aura" +name = "sp-consensus-babe" version = "2.0.0" dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", + "sc-application-crypto 2.0.0", + "sc-consensus-slots 2.0.0", + "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", "sp-timestamp 2.0.0", - "sr-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-version 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-aura-primitives 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-consensus-slots 2.0.0", - "substrate-executor 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-primitives 2.0.0", - "substrate-service 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-test-runtime-client 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-consensus-aura-primitives" +name = "sp-consensus-pow" version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-timestamp 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-inherents 2.0.0", + "sp-api 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-consensus-babe" +name = "sp-core" version = "2.0.0" dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fork-tree 2.0.0", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "ed25519-dalek 1.0.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "pdqselect 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sp-timestamp 2.0.0", - "sr-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-version 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-block-builder 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-consensus-slots 2.0.0", - "substrate-consensus-uncles 2.0.0", - "substrate-executor 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-primitives 2.0.0", - "substrate-service 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-test-runtime-client 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core-storage 2.0.0", + "sp-debug-derive 2.0.0", + "sp-externalities 2.0.0", + "sp-runtime-interface 2.0.0", + "sp-serializer 2.0.0", + "sp-std 2.0.0", + "substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "zeroize 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-consensus-babe-primitives" +name = "sp-core-storage" version = "2.0.0" dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-timestamp 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-consensus-slots 2.0.0", - "substrate-inherents 2.0.0", + "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-debug-derive 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-consensus-common" +name = "sp-debug-derive" version = "2.0.0" dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", - "substrate-test-runtime-client 2.0.0", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-consensus-pow" +name = "sp-externalities" version = "2.0.0" dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sp-timestamp 2.0.0", - "sr-primitives 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-consensus-pow-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", + "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core-storage 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-consensus-pow-primitives" +name = "sp-finality-granpda" version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "sc-application-crypto 2.0.0", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-consensus-slots" +name = "sp-finality-tracker" version = "2.0.0" dependencies = [ - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sr-primitives 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-test-runtime-client 2.0.0", + "sp-inherents 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-consensus-uncles" +name = "sp-inherents" version = "2.0.0" dependencies = [ - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-authorship 2.0.0", - "sr-primitives 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-inherents 2.0.0", - "substrate-primitives 2.0.0", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-debug-derive" +name = "sp-io" version = "2.0.0" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-externalities 2.0.0", + "sp-runtime-interface 2.0.0", + "sp-state-machine 2.0.0", + "sp-std 2.0.0", + "sp-trie 2.0.0", ] [[package]] -name = "substrate-executor" +name = "sp-keyring" version = "2.0.0" dependencies = [ - "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cranelift-codegen 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cranelift-entity 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cranelift-frontend 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cranelift-native 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cranelift-wasm 0.50.0 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-version 2.0.0", - "substrate-client-api 2.0.0", - "substrate-externalities 2.0.0", - "substrate-offchain 2.0.0", - "substrate-panic-handler 2.0.0", - "substrate-primitives 2.0.0", - "substrate-runtime-interface 2.0.0", - "substrate-runtime-test 2.0.0", - "substrate-serializer 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-test-runtime 2.0.0", - "substrate-trie 2.0.0", - "substrate-wasm-interface 2.0.0", - "test-case 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmtime-environ 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmtime-jit 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmtime-runtime 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "strum 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-externalities" +name = "sp-offchain" version = "2.0.0" dependencies = [ - "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-primitives-storage 2.0.0", + "sp-api 2.0.0", + "sp-runtime 2.0.0", ] [[package]] -name = "substrate-finality-grandpa" +name = "sp-panic-handler" version = "2.0.0" dependencies = [ - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "finality-grandpa 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fork-tree 2.0.0", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sp-finality-tracker 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-finality-grandpa-primitives 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-test-runtime-client 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-finality-grandpa-primitives" +name = "sp-phragmen" version = "2.0.0" dependencies = [ - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-application-crypto 2.0.0", + "sp-io 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", + "substrate-test-utils 2.0.0", ] [[package]] -name = "substrate-frame-rpc-support" +name = "sp-rpc" version = "2.0.0" dependencies = [ - "frame-support 2.0.0", - "frame-system 2.0.0", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-client-transports 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sc-rpc-api 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-primitives-storage 2.0.0", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", ] [[package]] -name = "substrate-frame-rpc-system" +name = "sp-runtime" version = "2.0.0" dependencies = [ - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "frame-system-rpc-runtime-api 2.0.0", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-core-client 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-derive 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sc-transaction-pool 2.0.0", + "paste 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sp-transaction-pool-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-client 2.0.0", - "substrate-primitives 2.0.0", - "substrate-test-runtime-client 2.0.0", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-arithmetic 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-inherents" +name = "sp-runtime-interface" version = "2.0.0" dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-executor 2.0.0", + "sp-core 2.0.0", + "sp-externalities 2.0.0", + "sp-io 2.0.0", + "sp-runtime-interface-proc-macro 2.0.0", + "sp-runtime-interface-test-wasm 2.0.0", + "sp-state-machine 2.0.0", + "sp-std 2.0.0", + "sp-wasm-interface 2.0.0", + "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-keyring" +name = "sp-runtime-interface-proc-macro" version = "2.0.0" dependencies = [ - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "strum 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-primitives 2.0.0", + "Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustversion 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-externalities 2.0.0", + "sp-runtime-interface 2.0.0", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "trybuild 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-keystore" +name = "sp-runtime-interface-test-wasm" version = "2.0.0" dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-application-crypto 2.0.0", - "substrate-primitives 2.0.0", - "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-runtime-interface 2.0.0", + "sp-std 2.0.0", + "substrate-wasm-builder-runner 1.0.4", ] [[package]] -name = "substrate-network" +name = "sp-sandbox" version = "2.0.0" dependencies = [ - "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "erased-serde 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "fork-tree 2.0.0", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", - "linked-hash-map 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "linked_hash_set 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "lru 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "quickcheck 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "slog_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sr-arithmetic 2.0.0", - "sr-primitives 2.0.0", - "substrate-block-builder 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-keyring 2.0.0", - "substrate-peerset 2.0.0", - "substrate-primitives 2.0.0", - "substrate-test-client 2.0.0", - "substrate-test-runtime 2.0.0", - "substrate-test-runtime-client 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "unsigned-varint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "zeroize 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-io 2.0.0", + "sp-std 2.0.0", + "wabt 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-offchain" +name = "sp-serializer" version = "2.0.0" dependencies = [ - "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.12.35 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper-rustls 0.17.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "sc-transaction-pool 2.0.0", - "sp-transaction-pool-api 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-client-api 2.0.0", - "substrate-client-db 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-offchain-primitives 2.0.0", - "substrate-primitives 2.0.0", - "substrate-test-runtime-client 2.0.0", - "threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-offchain-primitives" +name = "sp-sesssion" version = "2.0.0" dependencies = [ - "sr-api 2.0.0", - "sr-primitives 2.0.0", + "sp-api 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-panic-handler" +name = "sp-staking" version = "2.0.0" dependencies = [ - "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-peerset" +name = "sp-state-machine" version = "2.0.0" dependencies = [ - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "historical-data 2.0.0", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-externalities 2.0.0", + "sp-panic-handler 2.0.0", + "sp-trie 2.0.0", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-phragmen" +name = "sp-std" +version = "2.0.0" + +[[package]] +name = "sp-timestamp" version = "2.0.0" dependencies = [ - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "substrate-test-utils 2.0.0", + "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-inherents 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-primitives" +name = "sp-transaction-pool-api" version = "2.0.0" dependencies = [ - "base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "ed25519-dalek 1.0.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-debug-derive 2.0.0", - "substrate-externalities 2.0.0", - "substrate-primitives-storage 2.0.0", - "substrate-runtime-interface 2.0.0", - "substrate-serializer 2.0.0", - "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "zeroize 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-runtime 2.0.0", ] [[package]] -name = "substrate-primitives-storage" +name = "sp-transaction-pool-runtime-api" version = "2.0.0" dependencies = [ - "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-debug-derive 2.0.0", + "sp-api 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", ] [[package]] -name = "substrate-rpc" +name = "sp-trie" version = "2.0.0" dependencies = [ - "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-pubsub 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sc-rpc-api 2.0.0", - "sc-transaction-pool 2.0.0", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sp-transaction-pool-api 2.0.0", - "sr-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-version 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-executor 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-primitives 2.0.0", - "substrate-rpc-primitives 2.0.0", - "substrate-session 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-test-runtime-client 2.0.0", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-std 2.0.0", + "trie-bench 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-standardmap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-rpc-primitives" +name = "sp-version" version = "2.0.0" dependencies = [ + "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-primitives 2.0.0", + "sp-runtime 2.0.0", + "sp-std 2.0.0", ] [[package]] -name = "substrate-rpc-servers" +name = "sp-wasm-interface" version = "2.0.0" dependencies = [ - "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-http-server 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-pubsub 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "jsonrpc-ws-server 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", + "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-runtime-interface" -version = "2.0.0" +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "stable_deref_trait" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "stream-cipher" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-io 2.0.0", - "sr-std 2.0.0", - "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-executor 2.0.0", - "substrate-externalities 2.0.0", - "substrate-primitives 2.0.0", - "substrate-runtime-interface-proc-macro 2.0.0", - "substrate-runtime-interface-test-wasm 2.0.0", - "substrate-state-machine 2.0.0", - "substrate-wasm-interface 2.0.0", + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-runtime-interface-proc-macro" -version = "2.0.0" +name = "string" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustversion 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-externalities 2.0.0", - "substrate-runtime-interface 2.0.0", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "trybuild 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-runtime-interface-test-wasm" -version = "2.0.0" +name = "string-interner" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "sr-io 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", - "substrate-runtime-interface 2.0.0", - "substrate-wasm-builder-runner 1.0.4", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-runtime-test" -version = "2.0.0" +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "structopt" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-sandbox 2.0.0", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", - "substrate-wasm-builder-runner 1.0.4", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt-derive 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-serializer" -version = "2.0.0" +name = "structopt-derive" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", + "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-error 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-service" -version = "2.0.0" +name = "strum" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "derive_more 0.99.2 (registry+https://github.com/rust-lang/crates.io-index)", - "exit-future 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "grafana-data-source 2.0.0", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-multiaddr 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sc-transaction-pool 2.0.0", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-blockchain 2.0.0", - "sp-transaction-pool-api 2.0.0", - "sp-transaction-pool-runtime-api 2.0.0", - "sr-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-chain-spec 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-client-db 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-executor 2.0.0", - "substrate-finality-grandpa 2.0.0", - "substrate-finality-grandpa-primitives 2.0.0", - "substrate-keystore 2.0.0", - "substrate-network 2.0.0", - "substrate-offchain 2.0.0", - "substrate-primitives 2.0.0", - "substrate-rpc 2.0.0", - "substrate-rpc-servers 2.0.0", - "substrate-session 2.0.0", - "substrate-telemetry 2.0.0", - "substrate-test-runtime-client 2.0.0", - "substrate-tracing 2.0.0", - "sysinfo 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)", - "target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-executor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-timer 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "strum_macros 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-service-test" -version = "2.0.0" +name = "strum_macros" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fdlimit 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-transaction-pool-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-client 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-network 2.0.0", - "substrate-primitives 2.0.0", - "substrate-service 2.0.0", - "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", + "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-session" +name = "subkey" version = "2.0.0" dependencies = [ - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "frame-system 2.0.0", + "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "node-primitives 2.0.0", + "node-runtime 2.0.0", + "pallet-balances 2.0.0", + "pallet-transaction-payment 2.0.0", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-state-db" -version = "2.0.0" +name = "substrate-bip39" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-primitives 2.0.0", + "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-state-machine" +name = "substrate-build-script-utils" +version = "2.0.0" + +[[package]] +name = "substrate-frame-rpc-support" version = "2.0.0" dependencies = [ - "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "historical-data 2.0.0", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "frame-support 2.0.0", + "frame-system 2.0.0", + "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-client-transports 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-externalities 2.0.0", - "substrate-panic-handler 2.0.0", - "substrate-primitives 2.0.0", - "substrate-trie 2.0.0", - "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-rpc-api 2.0.0", + "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core-storage 2.0.0", + "tokio 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] -name = "substrate-telemetry" +name = "substrate-frame-rpc-system" version = "2.0.0" dependencies = [ - "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "frame-system-rpc-runtime-api 2.0.0", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures-timer 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libp2p 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-core-client 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "jsonrpc-derive 14.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client 2.0.0", + "sc-transaction-pool 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "slog-async 2.3.0 (git+https://github.com/paritytech/slog-async)", - "slog-json 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio-io 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-blockchain 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", + "sp-transaction-pool-api 2.0.0", + "substrate-test-runtime-client 2.0.0", ] [[package]] @@ -6572,16 +6613,16 @@ dependencies = [ "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-client-db 2.0.0", + "sc-executor 2.0.0", "sp-blockchain 2.0.0", - "sr-primitives 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-client-db 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-executor 2.0.0", - "substrate-keyring 2.0.0", - "substrate-primitives 2.0.0", - "substrate-state-machine 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-keyring 2.0.0", + "sp-runtime 2.0.0", + "sp-state-machine 2.0.0", ] [[package]] @@ -6589,10 +6630,10 @@ name = "substrate-test-primitives" version = "2.0.0" dependencies = [ "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-primitives 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", ] [[package]] @@ -6610,28 +6651,28 @@ dependencies = [ "pallet-babe 2.0.0", "pallet-timestamp 2.0.0", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-application-crypto 2.0.0", + "sc-client 2.0.0", + "sc-executor 2.0.0", "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-api 2.0.0", + "sp-block-builder 2.0.0", + "sp-consensus-aura 2.0.0", + "sp-consensus-babe 2.0.0", + "sp-core 2.0.0", + "sp-inherents 2.0.0", + "sp-io 2.0.0", + "sp-keyring 2.0.0", + "sp-offchain 2.0.0", + "sp-runtime 2.0.0", + "sp-runtime-interface 2.0.0", + "sp-sesssion 2.0.0", + "sp-state-machine 2.0.0", + "sp-std 2.0.0", "sp-transaction-pool-runtime-api 2.0.0", - "sr-api 2.0.0", - "sr-io 2.0.0", - "sr-primitives 2.0.0", - "sr-std 2.0.0", - "sr-version 2.0.0", - "substrate-application-crypto 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-client 2.0.0", - "substrate-consensus-aura-primitives 2.0.0", - "substrate-consensus-babe-primitives 2.0.0", - "substrate-executor 2.0.0", - "substrate-inherents 2.0.0", - "substrate-keyring 2.0.0", - "substrate-offchain-primitives 2.0.0", - "substrate-primitives 2.0.0", - "substrate-runtime-interface 2.0.0", - "substrate-session 2.0.0", - "substrate-state-machine 2.0.0", + "sp-trie 2.0.0", + "sp-version 2.0.0", "substrate-test-runtime-client 2.0.0", - "substrate-trie 2.0.0", "substrate-wasm-builder-runner 1.0.4", "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -6642,12 +6683,12 @@ version = "2.0.0" dependencies = [ "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-block-builder 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", "sp-blockchain 2.0.0", - "sr-primitives 2.0.0", - "substrate-block-builder 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-primitives 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", "substrate-test-client 2.0.0", "substrate-test-runtime 2.0.0", ] @@ -6656,39 +6697,6 @@ dependencies = [ name = "substrate-test-utils" version = "2.0.0" -[[package]] -name = "substrate-tracing" -version = "2.0.0" -dependencies = [ - "erased-serde 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "grafana-data-source 2.0.0", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)", - "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", - "substrate-telemetry 2.0.0", - "tracing 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "tracing-core 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "substrate-trie" -version = "2.0.0" -dependencies = [ - "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "sr-std 2.0.0", - "substrate-primitives 2.0.0", - "trie-bench 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-standardmap 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "substrate-wasm-builder" version = "1.0.8" @@ -6707,14 +6715,6 @@ dependencies = [ name = "substrate-wasm-builder-runner" version = "1.0.4" -[[package]] -name = "substrate-wasm-interface" -version = "2.0.0" -dependencies = [ - "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "subtle" version = "1.0.0" @@ -7226,16 +7226,16 @@ version = "0.0.1" dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sc-cli 2.0.0", + "sc-client 2.0.0", + "sc-client-api 2.0.0", + "sc-service 2.0.0", + "sp-api 2.0.0", + "sp-block-builder 2.0.0", "sp-blockchain 2.0.0", - "sr-api 2.0.0", - "sr-primitives 2.0.0", - "substrate-block-builder-runtime-api 2.0.0", - "substrate-cli 2.0.0", - "substrate-client 2.0.0", - "substrate-client-api 2.0.0", - "substrate-consensus-common 2.0.0", - "substrate-primitives 2.0.0", - "substrate-service 2.0.0", + "sp-consensus 2.0.0", + "sp-core 2.0.0", + "sp-runtime 2.0.0", ] [[package]] From 9c09b7826d092212160ebb22fcf7d8eebffea8c7 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Fri, 6 Dec 2019 18:19:50 +0100 Subject: [PATCH 090/129] fix crate names for bench and fuzz --- primitives/state-machine/benches/bench.rs | 2 +- primitives/state-machine/fuzz/Cargo.toml | 4 ++-- .../state-machine/fuzz/fuzz_targets/fuzz_transactions.rs | 2 +- .../fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs | 2 +- primitives/state-machine/fuzz/src/lib.rs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/primitives/state-machine/benches/bench.rs b/primitives/state-machine/benches/bench.rs index f8c8787e1e1f0..665152d7e90d3 100644 --- a/primitives/state-machine/benches/bench.rs +++ b/primitives/state-machine/benches/bench.rs @@ -16,7 +16,7 @@ use criterion::{Criterion, black_box, Bencher}; use criterion::{criterion_group, criterion_main}; -use substrate_state_machine::OverlayedChanges; +use sp_state_machine::OverlayedChanges; const CONTENT_KEY_SIZE: usize = 32; diff --git a/primitives/state-machine/fuzz/Cargo.toml b/primitives/state-machine/fuzz/Cargo.toml index b1b482c60adcc..d8ecf770b59a7 100644 --- a/primitives/state-machine/fuzz/Cargo.toml +++ b/primitives/state-machine/fuzz/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "substrate-state-machine-fuzz" +name = "sp-state-machine-fuzz" version = "0.0.1" authors = [] publish = false @@ -9,7 +9,7 @@ edition = "2018" [package.metadata] cargo-fuzz = true -[dependencies.substrate-state-machine] +[dependencies.sp-state-machine] path = ".." features = ["test-helpers"] diff --git a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs index 4de0b91feb5e1..561fb38d4ced6 100644 --- a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs +++ b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs @@ -3,5 +3,5 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - substrate_state_machine_fuzz::fuzz_transactions(data) + sp_state_machine_fuzz::fuzz_transactions(data) }); diff --git a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs index cd80c98a17257..07f673627c968 100644 --- a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs +++ b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs @@ -3,5 +3,5 @@ use libfuzzer_sys::fuzz_target; fuzz_target!(|data: &[u8]| { - substrate_state_machine_fuzz::fuzz_transactions_then_compactness(data) + sp_state_machine_fuzz::fuzz_transactions_then_compactness(data) }); diff --git a/primitives/state-machine/fuzz/src/lib.rs b/primitives/state-machine/fuzz/src/lib.rs index cf1ee726d73b8..9ea6eaeeded6b 100644 --- a/primitives/state-machine/fuzz/src/lib.rs +++ b/primitives/state-machine/fuzz/src/lib.rs @@ -15,7 +15,7 @@ // along with Substrate. If not, see . //! Substrate state machine fuzzing implementations. -use substrate_state_machine::OverlayedChanges; +use sp_state_machine::OverlayedChanges; use std::collections::HashMap; /// Size of key, max 255 From 1e1e790f9c6874ffc8cafb5b3b3550971bee72f3 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 9 Dec 2019 14:36:43 +0100 Subject: [PATCH 091/129] Address nits: remove test-helpers specific derive, make 'len' function test-helpers only. --- primitives/state-machine/Cargo.toml | 4 +++- primitives/state-machine/src/overlayed_changes.rs | 1 - utils/historical-data/src/lib.rs | 10 ++++------ utils/historical-data/src/synch_linear_transaction.rs | 3 +-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 823a63dd65c0a..cf49ccb0e1477 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -32,4 +32,6 @@ harness = false [features] default = [] -test-helpers = [] +test-helpers = [ + "historical-data/test-helpers", +] diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 5376413bff2b1..8b6dddbdc1e3c 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -71,7 +71,6 @@ pub struct OverlayedChangeSet { #[cfg(test)] impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { - let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = History::default(); diff --git a/utils/historical-data/src/lib.rs b/utils/historical-data/src/lib.rs index 2765531845d74..fd9c2b9d46190 100644 --- a/utils/historical-data/src/lib.rs +++ b/utils/historical-data/src/lib.rs @@ -22,8 +22,7 @@ pub mod synch_linear_transaction; /// An entry at a given history index. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] pub struct HistoricalValue { /// The stored value. pub value: V, @@ -43,8 +42,7 @@ impl HistoricalValue { } } -#[derive(PartialEq)] -#[cfg_attr(any(test, feature = "test"), derive(Debug))] +#[derive(Debug, PartialEq)] /// Results from cleaning a data with history. /// It should be used to update from the calling context, /// for instance remove this data from a map if it was cleared. @@ -55,8 +53,7 @@ pub enum CleaningResult { } /// History of value and their state. -#[derive(Debug, Clone)] -#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +#[derive(Debug, Clone, PartialEq)] pub struct History(pub(crate) smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>); impl Default for History { @@ -71,6 +68,7 @@ impl Default for History { const ALLOCATED_HISTORY: usize = 2; impl History { + #[cfg(any(test, feature = "test-helpers"))] /// Get current number of stored historical values. pub fn len(&self) -> usize { self.0.len() diff --git a/utils/historical-data/src/synch_linear_transaction.rs b/utils/historical-data/src/synch_linear_transaction.rs index 2a7be23481faf..058d547877f17 100644 --- a/utils/historical-data/src/synch_linear_transaction.rs +++ b/utils/historical-data/src/synch_linear_transaction.rs @@ -17,7 +17,6 @@ //! Linear arrangement of historical data with transactional //! support. //! -//! //! Described as a synchronized module as it contains historical //! data that do not require a separated state for most operation. //! Global state operation must therefore apply on all local values @@ -78,7 +77,7 @@ impl States { self.current_layer = 1; } - /// After a prospective was discarded, clear prospective history. + /// After a prospective was discarded, clear prospective history. pub fn apply_discard_prospective(value: &mut History) -> CleaningResult { if value.0.len() == 0 { return CleaningResult::Cleared; From 19733c54280a77e43dd5e57337600fa3d8088a57 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 11 Dec 2019 09:50:35 +0100 Subject: [PATCH 092/129] fix merge (runtime-io to sp-io) --- frame/support/src/storage/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 78af0fac1c4af..86ab5a8c8cbb9 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -31,12 +31,12 @@ pub mod generator; /// the transactional layer get reverted; otherwhise /// it is committed. pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { - runtime_io::storage::start_transaction(); + sp_io::storage::start_transaction(); let r = f(); if r.is_ok() { - runtime_io::storage::commit_transaction(); + sp_io::storage::commit_transaction(); } else { - runtime_io::storage::discard_transaction(); + sp_io::storage::discard_transaction(); } r } From fd262d900d371e8fbd1c6df3e6832b8ccb234eeb Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sat, 14 Dec 2019 18:31:11 +0100 Subject: [PATCH 093/129] Historical values documentation update. --- frame/support/src/storage/mod.rs | 13 ++-- primitives/externalities/src/lib.rs | 2 +- primitives/historical-data/README.md | 9 +-- primitives/historical-data/src/lib.rs | 28 +++++-- .../src/synch_linear_transaction.rs | 77 ++++++++++--------- test-utils/runtime/src/system.rs | 2 +- 6 files changed, 73 insertions(+), 58 deletions(-) diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 86ab5a8c8cbb9..ad7d891ae3566 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -28,17 +28,16 @@ pub mod generator; /// Execute under a transactional layer. /// /// If the result of execution is an error, -/// the transactional layer get reverted; otherwhise +/// the transactional layer get reverted; otherwise /// it is committed. pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { sp_io::storage::start_transaction(); - let r = f(); - if r.is_ok() { - sp_io::storage::commit_transaction(); - } else { - sp_io::storage::discard_transaction(); + let result = f(); + match result { + Ok(_r) => sp_io::storage::commit_transaction(), + Err(_e) => {sp_io::storage::discard_transaction(), } - r + result } /// A trait for working with macro-generated storage values under the substrate storage API. diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 507a5b74988a0..9bfc0d0dab6dc 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -161,7 +161,7 @@ pub trait Externalities: ExtensionStore { fn storage_start_transaction(&mut self); /// Discard a transactional layer, pending changes of every transaction behind this layer are - /// dropped (including committed changes) . + /// Dropped (including committed changes). fn storage_discard_transaction(&mut self); /// Commit a transactional layer. The changes stay attached to parent transaction layer. diff --git a/primitives/historical-data/README.md b/primitives/historical-data/README.md index c335b166d7a5b..e3af177ab3390 100644 --- a/primitives/historical-data/README.md +++ b/primitives/historical-data/README.md @@ -4,14 +4,11 @@ Crate with functionality to manage data that stores its own history. This covers: - linear history driven data, eg. transactional layers for overlay. -- long term storage with multiple branch, eg. offchain storage. -General design is container where query and update requires global -history context. +Design for this crate is to store history of each item in its own container. +Query and update actions can requires requires a global historical state. -History is serialize as a per item basis. - -This crates is `no_std` compatible as long as the `std` feature is not enabled. +This crate is `no_std` compatible as long as the `std` feature is not enabled. For more information see diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index fd9c2b9d46190..88e2f1c9fe8ae 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -43,16 +43,27 @@ impl HistoricalValue { } #[derive(Debug, PartialEq)] -/// Results from cleaning a data with history. -/// It should be used to update from the calling context, -/// for instance remove this data from a map if it was cleared. +/// The results from cleaning a historical value. +/// It should be used to update the calling context, +/// for instance if the historical value was stored +/// into a hashmap then it should be removed for +/// a `Cleared` result. pub enum CleaningResult { + /// No inner data was changed, even technical + /// data, therefore no update is needed. Unchanged, + /// Any data got modified, therefore an + /// update may be needed. Changed, + /// No data is stored anymore in this historical + /// value, it can be dropped. Cleared, } -/// History of value and their state. +/// History of value, this is used to keep trace of all changes +/// that occures on a value. +/// The different states for this value, are ordered by change time +/// in a simple stack. #[derive(Debug, Clone, PartialEq)] pub struct History(pub(crate) smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>); @@ -63,8 +74,8 @@ impl Default for History { } /// Size of preallocated history per element. -/// Currently at two for committed and prospective only. -/// It means that using transaction in a module got a direct allocation cost. +/// Current size is set to two, it is related to a use case +/// where value got two initial state by default (committed and prospective). const ALLOCATED_HISTORY: usize = 2; impl History { @@ -75,7 +86,7 @@ impl History { } #[cfg(any(test, feature = "test-helpers"))] - /// Create an history from an existing history. + /// Create an history from a sequence of historical values. pub fn from_iter(input: impl IntoIterator>) -> Self { let mut history = History::default(); for v in input { @@ -86,6 +97,9 @@ impl History { /// Push a value without checking if it can overwrite the current /// state. + /// This should only use after checking the state is correct + /// (last item of historical value got a smaller index than + /// the new one). pub fn push_unchecked(&mut self, item: HistoricalValue) { self.0.push(item); } diff --git a/primitives/historical-data/src/synch_linear_transaction.rs b/primitives/historical-data/src/synch_linear_transaction.rs index 058d547877f17..232a7c787a136 100644 --- a/primitives/historical-data/src/synch_linear_transaction.rs +++ b/primitives/historical-data/src/synch_linear_transaction.rs @@ -17,24 +17,26 @@ //! Linear arrangement of historical data with transactional //! support. //! -//! Described as a synchronized module as it contains historical -//! data that do not require a separated state for most operation. -//! Global state operation must therefore apply on all local values -//! with history synchronously. +//! This is mainly synchronous, it contains historical, +//! most operations do not require a separated state and +//! global state operation do update value synchronously +//! (to gain time on those operation a different design could +//! only change global state and apply garbage collection later). //! //! # Global state //! -//! The only global state is a counter of overlayed transaction layer. +//! The global state is the current numbre of overlayed transaction layer. //! Committing or discarding a layer must use this counter. //! //! # Local state //! -//! Local state is either a committed state (this is a single first independant level -//! of transaction) or a reference to the transaction counter in use in time of creation. +//! Local state is eitheir defined as `Committed` or the index of the number +//! of layer at the time of creation. +//! Transaction operation does not interfere with value in `Committed` state. use crate::CleaningResult; -/// Global state is a simple counter to the current overlay layer index. +/// Global state is the current number of overlay layers. #[derive(Debug, Clone, PartialEq, Eq)] pub struct States { current_layer: usize } @@ -52,7 +54,7 @@ impl States { State::Transaction(self.current_layer) } - /// Build any state for testing only. + /// Instantiate a random state, only for testing. #[cfg(any(test, feature = "test-helpers"))] pub fn test_state( current_layer: usize, @@ -60,23 +62,22 @@ impl States { States { current_layer } } + /// Update states when discarding prospective changes. + /// A subsequent update of all related stored history is needed. + pub fn discard_prospective(&mut self) { + self.current_layer = 1; + } + /// By default the current transaction is at least 1. - /// 0 can be use only to apply transaction change, in - /// this case transaction need to be restored to a valid - /// state afterward. + /// 0 is a transient state used when applying transaction change, + /// in this case transaction need to be restored to a valid state afterward. + /// This function does restore the state. pub fn finalize_discard(&mut self) { if self.current_layer == 0 { self.current_layer = 1; } } - /// Discard prospective changes to state. - /// It does not revert actual values. - /// A subsequent update of all related stored history is needed. - pub fn discard_prospective(&mut self) { - self.current_layer = 1; - } - /// After a prospective was discarded, clear prospective history. pub fn apply_discard_prospective(value: &mut History) -> CleaningResult { if value.0.len() == 0 { @@ -93,13 +94,15 @@ impl States { CleaningResult::Changed } - /// Commit prospective changes to state. + /// Update states when committing prospective. /// A subsequent update of all related stored history is needed. pub fn commit_prospective(&mut self) { self.current_layer = 1; } - /// After a prospective was committed, commit pending value and clear existing history. + /// After updating states to prospective, this function must be use + /// on all values from this prospective. It commits pending value and + /// clear existing history. pub fn apply_commit_prospective(value: &mut History) -> CleaningResult { if value.0.len() == 0 { return CleaningResult::Cleared; @@ -124,16 +127,15 @@ impl States { } /// Discard a transactional layer. - /// It does not revert actual values. - /// A subsequent update of all related stored history is needed. + /// A subsequent synchronous update of all related stored history is needed. pub fn discard_transaction(&mut self) { if self.current_layer > 0 { self.current_layer -= 1; } } - /// Update a value to previous transaction. - /// Multiple discard can be applied at the same time. + /// Apply transaction discard on a historical value. + /// Multiple calls to `discard_transaction` can be applied at once. pub fn apply_discard_transaction(&self, value: &mut History) -> CleaningResult { let init_len = value.0.len(); for i in (0 .. value.0.len()).rev() { @@ -155,8 +157,7 @@ impl States { } } - /// Discard a transactional layer. - /// It does not revert actual values. + /// Commit a transactional layer. /// A subsequent update of all related stored history is needed. pub fn commit_transaction(&mut self) { if self.current_layer > 1 { @@ -164,9 +165,9 @@ impl States { } } - /// Update a value to be the best historical value - /// after one or more `commit_transaction` calls. - /// Multiple discard can be applied at the same time. + /// Apply transaction commit on a historical value. + /// Multiple calls to `commit_transaction` can be applied at once. + /// Committing value removes all unneeded states. pub fn apply_commit_transaction(&self, value: &mut History) -> CleaningResult { let mut new_value = None; for i in (0 .. value.0.len()).rev() { @@ -196,15 +197,15 @@ impl States { } } -/// State for a historical value in a transaction -/// synchronously managed. +/// Historical value state for multiple transaction +/// with an additional commited state. #[derive(Debug, Clone, PartialEq, Eq)] pub enum State { /// Committed, transactional action do not /// change this state. Committed, /// Value in a transaction, contains the current - /// number of transaction at creation. + /// number of transaction when value did change. Transaction(usize), } @@ -217,6 +218,7 @@ impl State { } } } + /// An entry at a given history height. pub type HistoricalValue = crate::HistoricalValue; @@ -225,7 +227,7 @@ pub type History = crate::History; impl History { - /// Set a value, it uses a global state as parameter. + /// Set a value, this use a global state as parameter. pub fn set(&mut self, states: &States, value: V) { if let Some(v) = self.0.last_mut() { debug_assert!(v.index.transaction_index().unwrap_or(0) <= states.current_layer, @@ -248,7 +250,7 @@ impl History { self.0.last().map(|h| &h.value) } - /// Extracts the latest value if there is one. + /// Extracts the latest pending value if there is one. pub fn into_pending(mut self) -> Option { if let Some(v) = self.0.pop() { Some(v.value) @@ -257,6 +259,8 @@ impl History { } } + /// Get latest prospective value, excludes + /// committed values. #[cfg(any(test, feature = "test-helpers"))] pub fn get_prospective(&self) -> Option<&V> { match self.0.get(0) { @@ -281,6 +285,7 @@ impl History { } } + /// Get latest committed value. #[cfg(any(test, feature = "test-helpers"))] pub fn get_committed(&self) -> Option<&V> { if let Some(HistoricalValue { @@ -306,7 +311,7 @@ impl History { } } - /// Returns mutable latest pending historical value. + /// Returns mutable handle on latest pending historical value. pub fn get_mut(&mut self) -> Option> { self.0.last_mut().map(|h| h.as_mut()) } diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 517f258caa4b4..bf3e3fd835123 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -94,7 +94,7 @@ pub fn take_block_number() -> Option { Number::take() } -/// Return 1 if test successfull, 0 otherwhise. +/// Return 1 if test successfull, 0 otherwise. pub fn test_transactions() -> u64 { let origin_value: Option = storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX); let result: Result<(), _> = storage::with_transaction(|| { From d1554ec005ad955bdabf669eafeb8bf45b003556 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Sat, 14 Dec 2019 21:06:39 +0100 Subject: [PATCH 094/129] Documentation change on overlay_change --- primitives/state-machine/src/ext.rs | 2 +- .../state-machine/src/overlayed_changes.rs | 64 +++++++++---------- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index adf7a78442842..e30ff14047eb3 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -723,7 +723,7 @@ mod tests { #[test] fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { let mut overlay = prepare_overlay_with_changes(); - let conf = overlay.remove_changes_trie_config().unwrap(); + let conf = overlay.changes_trie_config.take().unwrap(); overlay.set_storage(vec![1], None); overlay.set_changes_trie_config(conf); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index e40287cdaf09d..5b56f2527a1c6 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -54,12 +54,9 @@ pub struct OverlayedValue { type TreeChangeSet = BTreeMap, History>; -/// Overlayed change set, keep history of values. +/// Overlayed change set, content is keeping trace of its history. /// -/// It stores hashmap containing a linear history of value. -/// Update on commit and discard operation (prospective or -/// overlay), are done in a synchronous manner by updating -/// all keys of the hashmap. +/// It uses a map containing a linear history of each values. #[derive(Debug, Clone, Default)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { @@ -68,6 +65,9 @@ pub struct OverlayedChangeSet { /// Top level storage changes. pub(crate) top: TreeChangeSet, /// Child storage changes. + /// OwnedChildInfo is currently an absolute value, for some child trie + /// operations (eg full deletion) it will need to change + /// to `History`. pub(crate) children: HashMap, (TreeChangeSet, OwnedChildInfo)>, } @@ -85,8 +85,9 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { } } -/// Variant of `set` value that update extrinsics. -/// It does remove latest dropped values. +/// Variant of historical data `set` value that also update extrinsics. +/// It avoid accessing two time the historical value item. +/// It does remove latest historical dropped items. fn set_with_extrinsic_overlayed_value( history: &mut History, states: &States, @@ -147,7 +148,7 @@ impl OverlayedChangeSet { self.top.is_empty() && self.children.is_empty() } - /// Discard prospective changes to state. + /// Discard prospective changes from the change set. pub fn discard_prospective(&mut self) { self.states.discard_prospective(); retain(&mut self.top, |_, history| States::apply_discard_prospective(history) != CleaningResult::Cleared); @@ -157,7 +158,7 @@ impl OverlayedChangeSet { }); } - /// Commit prospective changes to state. + /// Commit prospective changes into the change set. pub fn commit_prospective(&mut self) { self.states.commit_prospective(); retain(&mut self.top, |_, history| States::apply_commit_prospective(history) != CleaningResult::Cleared); @@ -173,7 +174,8 @@ impl OverlayedChangeSet { } /// Discard a transactional layer. - /// A transaction is always running (states always end with pending). + /// There is always a transactional layer running + /// (discarding the last trasactional layer open a new one). pub fn discard_transaction(&mut self) { self.states.discard_transaction(); let states = &self.states; @@ -185,7 +187,7 @@ impl OverlayedChangeSet { self.states.finalize_discard(); } - /// Commit a transactional layer. + /// Commit a transactional layer into previous transaction layer. pub fn commit_transaction(&mut self) { self.states.commit_transaction(); let states = &self.states; @@ -196,7 +198,7 @@ impl OverlayedChangeSet { }); } - /// Iterator over current state of a given overlay, including change trie information. + /// Iterator over current values of a given overlay, including change trie information. pub fn iter_overlay( &self, storage_key: Option<&[u8]>, @@ -219,7 +221,7 @@ impl OverlayedChangeSet { } - /// Iterator over current state of a given overlay, values only. + /// Iterator over current values of a given overlay. pub fn iter_values( &self, storage_key: Option<&[u8]>, @@ -228,7 +230,7 @@ impl OverlayedChangeSet { .map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) } - /// Iterator over current state of all children overlays, values only. + /// Iterator over current values of all children overlays. pub fn children_iter( &self, ) -> impl Iterator`. + /// Iterator over current values of all children overlays. + /// Variant of `children_iter` with owned `Vec` keys and values. pub fn owned_children_iter<'a>( &'a self, ) -> impl Iterator impl Iterator BTreeMap, OverlayedValue> { let mut result = BTreeMap::new(); @@ -302,8 +304,8 @@ impl OverlayedChangeSet { } /// Test only method to access current commited changes. - /// It is here to keep old test compatibility and should be - /// avoid for new tests. + /// This method is only here to keep compatibility with previous tests, + /// please do not use for new tests. #[cfg(test)] pub(crate) fn top_committed(&self) -> BTreeMap, OverlayedValue> { let mut result = BTreeMap::new(); @@ -314,7 +316,6 @@ impl OverlayedChangeSet { } result } - } impl OverlayedChanges { @@ -339,12 +340,6 @@ impl OverlayedChanges { true } - #[cfg(test)] - /// To allow value without extrinsic this can be use in test to disable change trie. - pub(crate) fn remove_changes_trie_config(&mut self) -> Option { - self.changes_trie_config.take() - } - /// Returns a double-Option: None if the key is unknown (i.e. and the query should be refered /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. @@ -516,8 +511,8 @@ impl OverlayedChanges { self.set_storage(EXTRINSIC_INDEX.to_vec(), Some(extrinsic_index.encode())); } - /// Test only method to build from committed info and prospective. - /// Create an history of two states. + /// Test only method to create a change set from committed + /// and prospective content. #[cfg(test)] pub(crate) fn new_from_top( committed: Vec<(Vec, Option>)>, @@ -552,7 +547,7 @@ impl OverlayedChanges { } #[cfg(any(test, feature = "test-helpers"))] - /// Iterator over current state of the overlay. + /// Iterator over current values of the overlay. pub fn iter_values( &self, storage_key: Option<&[u8]>, @@ -561,8 +556,9 @@ impl OverlayedChanges { } #[cfg(any(test, feature = "test-helpers"))] - /// Count (slow) the number of key value, history included. - /// Only for debugging or testing usage. + /// Count the number of key value pairs, at every history states. + /// Should only be use for debugging or testing, this is slow + /// and technical. pub fn top_count_keyvalue_pair(&self) -> usize { let mut result = 0; for (_, v) in self.changes.top.iter() { From 81f5c9027b62a4f68793110e63d757356b5c3f72 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 17 Dec 2019 20:22:38 +0100 Subject: [PATCH 095/129] adding cargo.lock to overlay fuzzing --- primitives/state-machine/fuzz/Cargo.lock | 1463 ++++++++++++++++++++++ 1 file changed, 1463 insertions(+) create mode 100644 primitives/state-machine/fuzz/Cargo.lock diff --git a/primitives/state-machine/fuzz/Cargo.lock b/primitives/state-machine/fuzz/Cargo.lock new file mode 100644 index 0000000000000..07fcc9c2b7617 --- /dev/null +++ b/primitives/state-machine/fuzz/Cargo.lock @@ -0,0 +1,1463 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ahash" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arbitrary" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "arrayref" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "backtrace" +version = "0.3.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base58" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "bitvec" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "blake2-rfc" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byte-slice-cast" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "c2-chacha" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cc" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clear_on_drop" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "const-random" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "const-random-macro" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "curve25519-dalek" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "curve25519-dalek" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.0-pre.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "curve25519-dalek 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "elastic-array" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "environmental" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "failure" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure_derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fixed-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "getrandom" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hash-db" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hashbrown" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hashbrown" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ahash 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "heapsize" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hex" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hmac" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hmac-drbg" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "impl-codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "impl-serde" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "keccak" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.65" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libfuzzer-sys" +version = "0.1.0" +source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#4ad88ec54fa2e7a0c877b6477cc86ea0de14a90d" +dependencies = [ + "arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libsecp256k1" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac-drbg 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lock_api" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lock_api" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "malloc_size_of_derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memory-db" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "merlin" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num-bigint" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-integer" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-rational" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "once_cell" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "parity-scale-codec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bitvec 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-slice-cast 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec-derive 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parity-util-mem" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "malloc_size_of_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parity-wasm" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "parking_lot" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parking_lot_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pbkdf2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "primitive-types" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "fixed-hash 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-codec 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "uint 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro2" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_chacha" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "regex" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-hex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "schnorrkel" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "curve25519-dalek 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "zeroize 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "scopeguard" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "scopeguard" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_derive" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sha2" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-core" +version = "2.0.0" +dependencies = [ + "base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "ed25519-dalek 1.0.0-pre.3 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-debug-derive 2.0.0", + "sp-externalities 2.0.0", + "sp-runtime-interface 2.0.0", + "sp-std 2.0.0", + "sp-storage 2.0.0", + "substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-debug-derive" +version = "2.0.0" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-externalities" +version = "2.0.0" +dependencies = [ + "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-std 2.0.0", + "sp-storage 2.0.0", +] + +[[package]] +name = "sp-historical-data" +version = "2.0.0" +dependencies = [ + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-std 2.0.0", +] + +[[package]] +name = "sp-panic-handler" +version = "2.0.0" +dependencies = [ + "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-runtime-interface" +version = "2.0.0" +dependencies = [ + "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-externalities 2.0.0", + "sp-runtime-interface-proc-macro 2.0.0", + "sp-std 2.0.0", + "sp-wasm-interface 2.0.0", + "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-runtime-interface-proc-macro" +version = "2.0.0" +dependencies = [ + "Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-state-machine" +version = "2.0.0" +dependencies = [ + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-externalities 2.0.0", + "sp-historical-data 2.0.0", + "sp-panic-handler 2.0.0", + "sp-trie 2.0.0", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-state-machine-fuzz" +version = "0.0.1" +dependencies = [ + "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", + "sp-state-machine 2.0.0", +] + +[[package]] +name = "sp-std" +version = "2.0.0" + +[[package]] +name = "sp-storage" +version = "2.0.0" +dependencies = [ + "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-debug-derive 2.0.0", + "sp-std 2.0.0", +] + +[[package]] +name = "sp-trie" +version = "2.0.0" +dependencies = [ + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sp-core 2.0.0", + "sp-std 2.0.0", + "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "sp-wasm-interface" +version = "2.0.0" +dependencies = [ + "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "substrate-bip39" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "subtle" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "syn" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synstructure" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tiny-bip39" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "once_cell 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "toml" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "trie-db" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "trie-root" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "twox-hash" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "typenum" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "uint" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wasi" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wasmi" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmi-validation 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasmi-validation" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "zeroize" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +"checksum ahash 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "2f00e10d4814aa20900e7948174384f79f1317f24f0ba7494e735111653fc330" +"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" +"checksum arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c7d1523aa3a127adf8b27af2404c03c12825b4c4d0698f01648d63fa9df62ee" +"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" +"checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +"checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" +"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" +"checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea" +"checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" +"checksum base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum bitvec 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" +"checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" +"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +"checksum byte-slice-cast 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f6209f3b2c1edea170002e016d5ead6903d3bb0a846477f53bbeb614967a52a9" +"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" +"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" +"checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe" +"checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59" +"checksum constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "995a44c877f9212528ccc74b21a232f66ad69001e40ede5bcee2ac9ef2657120" +"checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +"checksum curve25519-dalek 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" +"checksum curve25519-dalek 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" +"checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +"checksum ed25519-dalek 1.0.0-pre.3 (registry+https://github.com/rust-lang/crates.io-index)" = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" +"checksum elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "073be79b6538296faf81c631872676600616073817dd9a440c477ad09b408983" +"checksum environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "34f8467a0284de039e6bd0e25c14519538462ba5beb548bb1f03e645097837a8" +"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" +"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" +"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +"checksum fixed-hash 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72fe7539e2c5692c6989f2f9c0457e42f1e5768f96b85c87d273574670ae459f" +"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407" +"checksum hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" +"checksum hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" +"checksum hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3bae29b6653b3412c2e71e9d486db9f9df5d701941d86683005efb9f2d28e3da" +"checksum hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" +"checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" +"checksum hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" +"checksum hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +"checksum hmac-drbg 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" +"checksum impl-codec 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" +"checksum impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "58e3cae7e99c7ff5a995da2cf78dd0a5383740eda71d98cf7b1910c301ac69b8" +"checksum impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" +"checksum keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" +"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" +"checksum libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)" = "" +"checksum libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd9a7c16c9487e710536b699c962f022266347c94201174aa0a7eb0546051aa" +"checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" +"checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc" +"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +"checksum malloc_size_of_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e37c5d4cd9473c5f4c9c111f033f15d4df9bd378fdf615944e360a4f55a05f0b" +"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" +"checksum memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ef49315991403ba5fa225a70399df5e115f57b274cb0b1b4bcd6e734fa5bd783" +"checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" +"checksum merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "de2d16d3b15fec5943d1144f861f61f279d165fdd60998ca262913b9bf1c8adb" +"checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" +"checksum num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f9c3f34cdd24f334cb265d9bf8bfa8a241920d026916785747a92f0e55541a1a" +"checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" +"checksum num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2885278d5fe2adc2f75ced642d52d879bffaceb5a2e0b1d4309ffdfb239b454" +"checksum num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "443c53b3c3531dfcbfa499d8893944db78474ad7a1d87fa2d94d1a2231693ac6" +"checksum once_cell 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "532c29a261168a45ce28948f9537ddd7a5dd272cc513b3017b1e82a88f962c37" +"checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +"checksum parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f9f9d99dae413590a5f37e43cd99b94d4e62a244160562899126913ea7108673" +"checksum parity-scale-codec-derive 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "492ac3aa93d6caa5d20e4e3e0b75d08e2dcd9dd8a50d19529548b6fe11b3f295" +"checksum parity-util-mem 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "570093f39f786beea92dcc09e45d8aae7841516ac19a50431953ac82a0e8f85c" +"checksum parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" +"checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" +"checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +"checksum parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" +"checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +"checksum pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" +"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" +"checksum primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a0253db64c26d8b4e7896dd2063b516d2a1b9e0a5da26b5b78335f236d1e9522" +"checksum proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" +"checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" +"checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" +"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" +"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412" +"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" +"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +"checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" +"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" +"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" +"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" +"checksum rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "403bb3a286107a04825a5f82e1270acc1e14028d3d554d7a1e08914549575ab8" +"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +"checksum schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" +"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" +"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" +"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4b39bd9b0b087684013a792c59e3e07a46a01d2322518d8a1104641a0b1be0" +"checksum serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "ca13fc1a832f793322228923fbb3aba9f3f44444898f835d31ad1b74fa0a2bf8" +"checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" +"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +"checksum static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +"checksum substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" +"checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" +"checksum subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab3af2eb31c42e8f0ccf43548232556c42737e01a96db6e1777b0be108e79799" +"checksum syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "661641ea2aa15845cddeb97dad000d22070bb5c1fb456b96c1cba883ec691e92" +"checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" +"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c5676413eaeb1ea35300a0224416f57abc3bd251657e0fafc12c47ff98c060" +"checksum tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" +"checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" +"checksum trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "784a9813d23f18bccab728ab039c39b8a87d0d6956dcdece39e92f5cffe5076e" +"checksum trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0b779f7c1c8fe9276365d9d5be5c4b5adeacf545117bb3f64c974305789c5c0b" +"checksum twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" +"checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" +"checksum uint 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" +"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" +"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" +"checksum wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" +"checksum wasmi-validation 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum zeroize 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" +"checksum zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" From 70946d1dc796ab1f3fd149ef6f4512f5b9f36f48 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 2 Jan 2020 11:41:03 +0100 Subject: [PATCH 096/129] rewrite readme, rename historicalvalue to historicalentry --- frame/support/src/storage/mod.rs | 2 +- primitives/externalities/src/lib.rs | 11 +++-- primitives/historical-data/README.md | 15 ++++-- primitives/historical-data/src/lib.rs | 46 +++++++++---------- .../src/synch_linear_transaction.rs | 32 ++++++------- .../state-machine/src/changes_trie/build.rs | 16 +++---- primitives/state-machine/src/ext.rs | 6 +-- .../state-machine/src/overlayed_changes.rs | 8 ++-- 8 files changed, 72 insertions(+), 64 deletions(-) diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 138a784cb085f..40a90d86dd389 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -25,7 +25,7 @@ pub mod hashed; pub mod child; pub mod generator; -/// Execute under a transactional layer. +/// Execute under a new transactional layer. /// /// If the result of execution is an error, /// the transactional layer get reverted; otherwise diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 7aa29a16e50a8..4a783b620fb1a 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -203,14 +203,17 @@ pub trait Externalities: ExtensionStore { /// Returns the SCALE encoded hash. fn storage_changes_root(&mut self, parent: &[u8]) -> Result>, ()>; - /// Create a new transactional layer. + /// Start a new transactional layer. fn storage_start_transaction(&mut self); - /// Discard a transactional layer, pending changes of every transaction behind this layer are - /// Dropped (including committed changes). + /// Discard current transactional layer, pending changes of this transaction layer are + /// dropped. + /// The transactional layer is closed, a new one is started if this was the last layer. fn storage_discard_transaction(&mut self); - /// Commit a transactional layer. The changes stay attached to parent transaction layer. + /// Commit current transactional layer. + /// The changes stay attached to parent transaction layer and could still be discarded. + /// The transactional layer is closed, a new one is started if this was the last layer. fn storage_commit_transaction(&mut self); } diff --git a/primitives/historical-data/README.md b/primitives/historical-data/README.md index e3af177ab3390..4b14e37e1749b 100644 --- a/primitives/historical-data/README.md +++ b/primitives/historical-data/README.md @@ -1,14 +1,19 @@ ## Historical data -Crate with functionality to manage data that stores its own history. +Crate for managing data and their associated history. This covers: -- linear history driven data, eg. transactional layers for overlay. +- data with a sequential history, eg. transactional layers and their associated changes. -Design for this crate is to store history of each item in its own container. -Query and update actions can requires requires a global historical state. +This crate is storing single historical data, for instance in a sequential list of state. +Historical collection of data will therefore be preferably a collection of individual historical data. +For instance with historical data being a list of change, a collection of historical data will +be a collection of list rather than a list of collection. -This crate is `no_std` compatible as long as the `std` feature is not enabled. +For multiple individual data, we refer to a global historical state, +query and update actions are using it as an input parameter. + +This crate is `no_std` compatible if the `std` feature is not enabled. For more information see diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index 88e2f1c9fe8ae..afd0e6ce22935 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -21,24 +21,37 @@ pub mod synch_linear_transaction; -/// An entry at a given history index. +/// History of values, this is used to keep trace of all changes +/// that occures on a value. +/// The different states for this value, are ordered by change time +/// in a simple stack. +#[derive(Debug, Clone, PartialEq)] +pub struct History(pub(crate) smallvec::SmallVec<[HistoricalEntry; ALLOCATED_HISTORY]>); + +impl Default for History { + fn default() -> Self { + History(Default::default()) + } +} + +/// An entry of a historical data at a given history point. #[derive(Debug, Clone, PartialEq)] -pub struct HistoricalValue { +pub struct HistoricalEntry { /// The stored value. pub value: V, /// The moment in history when the value got set. pub index: I, } -impl From<(V, I)> for HistoricalValue { - fn from(input: (V, I)) -> HistoricalValue { - HistoricalValue { value: input.0, index: input.1 } +impl From<(V, I)> for HistoricalEntry { + fn from(input: (V, I)) -> HistoricalEntry { + HistoricalEntry { value: input.0, index: input.1 } } } -impl HistoricalValue { - fn as_mut(&mut self) -> HistoricalValue<&mut V, I> { - HistoricalValue { value: &mut self.value, index: self.index.clone() } +impl HistoricalEntry { + fn as_mut(&mut self) -> HistoricalEntry<&mut V, I> { + HistoricalEntry { value: &mut self.value, index: self.index.clone() } } } @@ -60,19 +73,6 @@ pub enum CleaningResult { Cleared, } -/// History of value, this is used to keep trace of all changes -/// that occures on a value. -/// The different states for this value, are ordered by change time -/// in a simple stack. -#[derive(Debug, Clone, PartialEq)] -pub struct History(pub(crate) smallvec::SmallVec<[HistoricalValue; ALLOCATED_HISTORY]>); - -impl Default for History { - fn default() -> Self { - History(Default::default()) - } -} - /// Size of preallocated history per element. /// Current size is set to two, it is related to a use case /// where value got two initial state by default (committed and prospective). @@ -87,7 +87,7 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] /// Create an history from a sequence of historical values. - pub fn from_iter(input: impl IntoIterator>) -> Self { + pub fn from_iter(input: impl IntoIterator>) -> Self { let mut history = History::default(); for v in input { history.push_unchecked(v); @@ -100,7 +100,7 @@ impl History { /// This should only use after checking the state is correct /// (last item of historical value got a smaller index than /// the new one). - pub fn push_unchecked(&mut self, item: HistoricalValue) { + pub fn push_unchecked(&mut self, item: HistoricalEntry) { self.0.push(item); } } diff --git a/primitives/historical-data/src/synch_linear_transaction.rs b/primitives/historical-data/src/synch_linear_transaction.rs index 232a7c787a136..57e9babfdbc22 100644 --- a/primitives/historical-data/src/synch_linear_transaction.rs +++ b/primitives/historical-data/src/synch_linear_transaction.rs @@ -36,6 +36,12 @@ use crate::CleaningResult; +/// An entry at a given history height. +pub type HistoricalEntry = crate::HistoricalEntry; + +/// History of value and their state. +pub type History = crate::History; + /// Global state is the current number of overlay layers. #[derive(Debug, Clone, PartialEq, Eq)] pub struct States { current_layer: usize } @@ -139,7 +145,7 @@ impl States { pub fn apply_discard_transaction(&self, value: &mut History) -> CleaningResult { let init_len = value.0.len(); for i in (0 .. value.0.len()).rev() { - if let HistoricalValue { + if let HistoricalEntry { value: _, index: State::Transaction(ix), } = value.0[i] { @@ -171,7 +177,7 @@ impl States { pub fn apply_commit_transaction(&self, value: &mut History) -> CleaningResult { let mut new_value = None; for i in (0 .. value.0.len()).rev() { - if let HistoricalValue { + if let HistoricalEntry { value: _, index: State::Transaction(ix), } = value.0[i] { @@ -187,7 +193,7 @@ impl States { } else { break } } if let Some(new_value) = new_value { - value.0.push(HistoricalValue { + value.0.push(HistoricalEntry { value: new_value, index: State::Transaction(self.current_layer), }); @@ -219,12 +225,6 @@ impl State { } } -/// An entry at a given history height. -pub type HistoricalValue = crate::HistoricalValue; - -/// History of value and their state. -pub type History = crate::History; - impl History { /// Set a value, this use a global state as parameter. @@ -239,7 +239,7 @@ impl History { return; } } - self.0.push(HistoricalValue { + self.0.push(HistoricalEntry { value, index: State::Transaction(states.current_layer), }); @@ -264,11 +264,11 @@ impl History { #[cfg(any(test, feature = "test-helpers"))] pub fn get_prospective(&self) -> Option<&V> { match self.0.get(0) { - Some(HistoricalValue { + Some(HistoricalEntry { value: _, index: State::Committed, }) => { - if let Some(HistoricalValue { + if let Some(HistoricalEntry { value, index: State::Transaction(_), }) = self.0.get(1) { @@ -277,7 +277,7 @@ impl History { None } }, - Some(HistoricalValue { + Some(HistoricalEntry { value, index: State::Transaction(_), }) => Some(&value), @@ -288,7 +288,7 @@ impl History { /// Get latest committed value. #[cfg(any(test, feature = "test-helpers"))] pub fn get_committed(&self) -> Option<&V> { - if let Some(HistoricalValue { + if let Some(HistoricalEntry { value, index: State::Committed, }) = self.0.get(0) { @@ -301,7 +301,7 @@ impl History { /// Extracts the committed value if there is one. pub fn into_committed(mut self) -> Option { self.0.truncate(1); - if let Some(HistoricalValue { + if let Some(HistoricalEntry { value, index: State::Committed, }) = self.0.pop() { @@ -312,7 +312,7 @@ impl History { } /// Returns mutable handle on latest pending historical value. - pub fn get_mut(&mut self) -> Option> { + pub fn get_mut(&mut self) -> Option> { self.0.last_mut().map(|h| h.as_mut()) } diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 514c5a70ff38f..d75bd60c83e1c 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -329,7 +329,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use sp_historical_data::synch_linear_transaction::{States, State, History, HistoricalValue}; + use sp_historical_data::synch_linear_transaction::{States, State, History, HistoricalEntry}; use super::*; const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); @@ -410,7 +410,7 @@ mod test { value: Some(3u32.encode()), extrinsics: None, }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), (vec![100], History::from_iter(vec![ (OverlayedValue { value: Some(vec![202]), @@ -420,19 +420,19 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), (vec![101], History::from_iter(vec![ (OverlayedValue { value: Some(vec![203]), extrinsics: Some(vec![1].into_iter().collect()) }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), (vec![103], History::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![0, 1].into_iter().collect()) }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), ].into_iter().collect(), children: vec![ (child_trie_key1, (vec![ @@ -445,7 +445,7 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), ].into_iter().collect(), CHILD_INFO_1.to_owned())), (child_trie_key2, (vec![ (vec![100], History::from_iter(vec![ @@ -453,7 +453,7 @@ mod test { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), ].into_iter().collect(), CHILD_INFO_2.to_owned())), ].into_iter().collect(), }, @@ -654,7 +654,7 @@ mod test { value: None, extrinsics: Some(vec![1].into_iter().collect()), }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))); + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))); let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; let changes_trie_nodes = prepare_input( diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 25961f2172826..dcbd9a65420aa 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -659,7 +659,7 @@ mod tests { }, backend::InMemory, overlayed_changes::{OverlayedValue, OverlayedChangeSet}, }; use sp_core::storage::{Storage, StorageChild}; - use sp_historical_data::synch_linear_transaction::{History, HistoricalValue, State}; + use sp_historical_data::synch_linear_transaction::{History, HistoricalEntry, State}; type TestBackend = InMemory; type TestChangesTrieStorage = InMemoryChangesTrieStorage; @@ -681,13 +681,13 @@ mod tests { value: Some(3u32.encode()), extrinsics: Some(vec![1].into_iter().collect()) }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), (vec![1], History::from_iter(vec![ (OverlayedValue { value: Some(vec![100]), extrinsics: Some(vec![1].into_iter().collect()) }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalValue { value, index }))), + ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), ].into_iter().collect(), }, } diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 056372cc55641..d3a600a15cb1a 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -23,7 +23,7 @@ use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; use sp_historical_data::synch_linear_transaction::{ - History, HistoricalValue, States, + History, HistoricalEntry, States, }; use sp_historical_data::CleaningResult; use std::ops; @@ -78,7 +78,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = History::default(); - history.push_unchecked(HistoricalValue { value, index: State::Committed }); + history.push_unchecked(HistoricalEntry { value, index: State::Committed }); history })).collect(); result @@ -120,7 +120,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.value.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - history.push_unchecked(HistoricalValue { + history.push_unchecked(HistoricalEntry { index: state, value: OverlayedValue { value, @@ -132,7 +132,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics: Option> = None; extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - history.push_unchecked(HistoricalValue { + history.push_unchecked(HistoricalEntry { index: state, value: OverlayedValue { value, From dc4462a6190c0cd2e592117e4a5eebe55047ee8b Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 2 Jan 2020 12:11:35 +0100 Subject: [PATCH 097/129] spelling an slight documentation change. --- frame/contracts/src/wasm/runtime.rs | 2 +- primitives/historical-data/src/lib.rs | 27 ++++++------ .../src/synch_linear_transaction.rs | 41 ++++++++++--------- .../state-machine/src/overlayed_changes.rs | 16 ++++---- 4 files changed, 43 insertions(+), 43 deletions(-) diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index cbf666dde2d67..7b6ecabfb414d 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -647,7 +647,7 @@ define_env!(Env, , // Record a request to restore the caller contract to the specified contract. // // At the finalization stage, i.e. when all changes from the extrinsic that invoked this - // contract are commited, this function will compute a tombstone hash from the caller's + // contract are committed, this function will compute a tombstone hash from the caller's // storage and the given code hash and if the hash matches the hash found in the tombstone at // the specified address - kill the caller contract and restore the destination contract and set // the specified `rent_allowance`. All caller's funds are transfered to the destination. diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index afd0e6ce22935..b931dc22b8106 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -21,13 +21,19 @@ pub mod synch_linear_transaction; -/// History of values, this is used to keep trace of all changes -/// that occures on a value. +/// History of values being used to keep trace of all changes +/// that occurs (all different state a value can be in depending +/// on the global state). /// The different states for this value, are ordered by change time /// in a simple stack. #[derive(Debug, Clone, PartialEq)] pub struct History(pub(crate) smallvec::SmallVec<[HistoricalEntry; ALLOCATED_HISTORY]>); +/// Size of preallocated history per element. +/// Current size is two, that is currently related to the use case +/// where value got `committed` and `prospective` initial state by default. +const ALLOCATED_HISTORY: usize = 2; + impl Default for History { fn default() -> Self { History(Default::default()) @@ -56,11 +62,10 @@ impl HistoricalEntry { } #[derive(Debug, PartialEq)] -/// The results from cleaning a historical value. -/// It should be used to update the calling context, -/// for instance if the historical value was stored -/// into a hashmap then it should be removed for -/// a `Cleared` result. +/// The results from changing a historical value. +/// It should be used to apply subsequent update the calling context. +/// For instance if the historical value was stored into a hashmap, +/// then it should be removed from it on a `Cleared` result. pub enum CleaningResult { /// No inner data was changed, even technical /// data, therefore no update is needed. @@ -68,16 +73,10 @@ pub enum CleaningResult { /// Any data got modified, therefore an /// update may be needed. Changed, - /// No data is stored anymore in this historical - /// value, it can be dropped. + /// No historical data is stored anymore, it can be dropped. Cleared, } -/// Size of preallocated history per element. -/// Current size is set to two, it is related to a use case -/// where value got two initial state by default (committed and prospective). -const ALLOCATED_HISTORY: usize = 2; - impl History { #[cfg(any(test, feature = "test-helpers"))] /// Get current number of stored historical values. diff --git a/primitives/historical-data/src/synch_linear_transaction.rs b/primitives/historical-data/src/synch_linear_transaction.rs index 57e9babfdbc22..8a175ad844dff 100644 --- a/primitives/historical-data/src/synch_linear_transaction.rs +++ b/primitives/historical-data/src/synch_linear_transaction.rs @@ -14,35 +14,37 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Linear arrangement of historical data with transactional +//! Linear sequence of historical data with transactional //! support. //! -//! This is mainly synchronous, it contains historical, -//! most operations do not require a separated state and -//! global state operation do update value synchronously -//! (to gain time on those operation a different design could -//! only change global state and apply garbage collection later). +//! This is mainly synchronous, global transactional state operation +//! do update value synchronously (to gain time on those operation a +//! different design could be to switch to only change global state +//! and apply garbage collection later if needed). //! //! # Global state //! -//! The global state is the current numbre of overlayed transaction layer. -//! Committing or discarding a layer must use this counter. +//! The global state is the current number of overlayed transaction layers. +//! Committing or discarding a layer simply change this counter and a +//! synchronous local state operations should follow to update every +//! associated historical data. //! //! # Local state //! -//! Local state is eitheir defined as `Committed` or the index of the number -//! of layer at the time of creation. +//! Local state is either defined as `Committed` or `Prospective`. +//! If in `Prospective` state, the index of the number of layesr at the time +//! of creation is also needed. //! Transaction operation does not interfere with value in `Committed` state. use crate::CleaningResult; -/// An entry at a given history height. -pub type HistoricalEntry = crate::HistoricalEntry; - /// History of value and their state. pub type History = crate::History; -/// Global state is the current number of overlay layers. +/// An entry at a given history height. +pub type HistoricalEntry = crate::HistoricalEntry; + +/// Global state contains only the current number of overlay layers. #[derive(Debug, Clone, PartialEq, Eq)] pub struct States { current_layer: usize } @@ -203,14 +205,14 @@ impl States { } } -/// Historical value state for multiple transaction -/// with an additional commited state. +/// Historical value state of multiple transaction +/// with an alterantive committed state. #[derive(Debug, Clone, PartialEq, Eq)] pub enum State { - /// Committed, transactional action do not - /// change this state. + /// Committed, in this state no action on transaction + /// can modify this value. Committed, - /// Value in a transaction, contains the current + /// Value relates to a transaction, the state contains the current /// number of transaction when value did change. Transaction(usize), } @@ -315,5 +317,4 @@ impl History { pub fn get_mut(&mut self) -> Option> { self.0.last_mut().map(|h| h.as_mut()) } - } diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index d3a600a15cb1a..13a6a0c063170 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -54,9 +54,9 @@ pub struct OverlayedValue { type TreeChangeSet = BTreeMap, History>; -/// Overlayed change set, content is keeping trace of its history. +/// Overlayed change set, content keep trace of its history. /// -/// It uses a map containing a linear history of each values. +/// Maps containing a linear history of each values are used. #[derive(Debug, Clone, Default)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { @@ -85,8 +85,8 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { } } -/// Variant of historical data `set` value that also update extrinsics. -/// It avoid accessing two time the historical value item. +/// Variant of historical data `set` function with internal extrinsics update. +/// It avoids accessing two times the historical value item. /// It does remove latest historical dropped items. fn set_with_extrinsic_overlayed_value( history: &mut History, @@ -198,7 +198,7 @@ impl OverlayedChangeSet { }); } - /// Iterator over current values of a given overlay, including change trie information. + /// Iterator over current values for a given overlay, including change trie information. pub fn iter_overlay( &self, storage_key: Option<&[u8]>, @@ -221,7 +221,7 @@ impl OverlayedChangeSet { } - /// Iterator over current values of a given overlay. + /// Iterator over current values for a given overlay. pub fn iter_values( &self, storage_key: Option<&[u8]>, @@ -252,7 +252,7 @@ impl OverlayedChangeSet { } /// Iterator over current values of all children overlays. - /// Variant of `children_iter` with owned `Vec` keys and values. + /// This is a variant of `children_iter` with owned `Vec` keys and values. pub fn owned_children_iter<'a>( &'a self, ) -> impl Iterator Date: Mon, 6 Jan 2020 18:30:25 +0100 Subject: [PATCH 098/129] Fix runtime tests. --- primitives/api/test/tests/runtime_calls.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index 440ea6551958a..c0e8baa8be0b8 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -134,7 +134,7 @@ fn use_trie_function() { fn use_history_data() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); let runtime_api = client.runtime_api(); - let block_id = BlockId::Number(client.info().chain.best_number); + let block_id = BlockId::Number(client.chain_info().best_number); assert!(runtime_api.use_history_data(&block_id).is_ok()); } @@ -142,7 +142,7 @@ fn use_history_data() { fn test_transactions() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); let runtime_api = client.runtime_api(); - let block_id = BlockId::Number(client.info().chain.best_number); + let block_id = BlockId::Number(client.chain_info().best_number); assert_eq!(runtime_api.use_transactions(&block_id).unwrap(), 1); } From 7a9950953752c7b50d90efedfa100eed4d3ec3d9 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 15:16:55 +0100 Subject: [PATCH 099/129] apply update-copyright.sh --- primitives/historical-data/src/lib.rs | 2 +- primitives/historical-data/src/synch_linear_transaction.rs | 2 +- primitives/state-machine/benches/bench.rs | 2 +- primitives/state-machine/fuzz/src/lib.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index b931dc22b8106..8620af2d6ca02 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. +// Copyright 2019-2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify diff --git a/primitives/historical-data/src/synch_linear_transaction.rs b/primitives/historical-data/src/synch_linear_transaction.rs index 8a175ad844dff..d61797bafeda0 100644 --- a/primitives/historical-data/src/synch_linear_transaction.rs +++ b/primitives/historical-data/src/synch_linear_transaction.rs @@ -1,4 +1,4 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. +// Copyright 2019-2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify diff --git a/primitives/state-machine/benches/bench.rs b/primitives/state-machine/benches/bench.rs index 665152d7e90d3..8cff06c525982 100644 --- a/primitives/state-machine/benches/bench.rs +++ b/primitives/state-machine/benches/bench.rs @@ -1,4 +1,4 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. +// Copyright 2019-2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Parity is free software: you can redistribute it and/or modify diff --git a/primitives/state-machine/fuzz/src/lib.rs b/primitives/state-machine/fuzz/src/lib.rs index 9ea6eaeeded6b..4ec2774c2dea5 100644 --- a/primitives/state-machine/fuzz/src/lib.rs +++ b/primitives/state-machine/fuzz/src/lib.rs @@ -1,4 +1,4 @@ -// Copyright 2019 Parity Technologies (UK) Ltd. +// Copyright 2019-2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify From 256f3a5c265f5fa2699d0d95ff3037a3976bdd6b Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 15:49:01 +0100 Subject: [PATCH 100/129] move transaction to its own transaction runtime module. --- frame/support/src/storage/mod.rs | 6 ++-- primitives/api/test/tests/runtime_calls.rs | 17 ----------- primitives/historical-data/src/lib.rs | 4 +-- primitives/io/src/lib.rs | 34 ++++++++++++---------- primitives/state-machine/src/basic.rs | 1 - test-utils/runtime/src/lib.rs | 20 +++++++++++++ 6 files changed, 44 insertions(+), 38 deletions(-) diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 4974a66ba11ed..e35ed9deb9770 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -31,11 +31,11 @@ pub mod generator; /// the transactional layer get reverted; otherwise /// it is committed. pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { - sp_io::storage::start_transaction(); + sp_io::transaction::start_transaction(); let result = f(); match result { - Ok(_) => sp_io::storage::commit_transaction(), - Err(_) => sp_io::storage::discard_transaction(), + Ok(_) => sp_io::transaction::commit_transaction(), + Err(_) => sp_io::transaction::discard_transaction(), } result } diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index c0e8baa8be0b8..16516ed038a6f 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -130,23 +130,6 @@ fn use_trie_function() { assert_eq!(runtime_api.use_trie(&block_id).unwrap(), 2); } -#[test] -fn use_history_data() { - let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); - let runtime_api = client.runtime_api(); - let block_id = BlockId::Number(client.chain_info().best_number); - assert!(runtime_api.use_history_data(&block_id).is_ok()); -} - -#[test] -fn test_transactions() { - let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::AlwaysWasm).build(); - let runtime_api = client.runtime_api(); - let block_id = BlockId::Number(client.chain_info().best_number); - assert_eq!(runtime_api.use_transactions(&block_id).unwrap(), 1); -} - - #[test] fn initialize_block_works() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::Both).build(); diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index 8620af2d6ca02..ec0c6e813fe14 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Data storage containing multiple states for a value. +//! Data storage containing multiple states of a value. //! This is used to store historical information for an item. #![cfg_attr(not(feature = "std"), no_std)] @@ -24,7 +24,7 @@ pub mod synch_linear_transaction; /// History of values being used to keep trace of all changes /// that occurs (all different state a value can be in depending /// on the global state). -/// The different states for this value, are ordered by change time +/// The different states of this value, are ordered by change time /// in a simple stack. #[derive(Debug, Clone, PartialEq)] pub struct History(pub(crate) smallvec::SmallVec<[HistoricalEntry; ALLOCATED_HISTORY]>); diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index cc6b700de6a80..5459a35990f8e 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -293,21 +293,6 @@ pub trait Storage { self.storage_changes_root(parent_hash).ok().and_then(|h| h) } - /// Start a new transaction. - fn start_transaction(&mut self) { - self.storage_start_transaction(); - } - - /// Discard a transactional layer. - fn discard_transaction(&mut self) { - self.storage_discard_transaction(); - } - - /// Commit a transactional layer. - fn commit_transaction(&mut self) { - self.storage_commit_transaction(); - } - /// Get the next key in storage after the given one in lexicographic order. fn next_key(&mut self, key: &[u8]) -> Option> { self.next_storage_key(&key) @@ -328,6 +313,25 @@ pub trait Storage { } } +/// Interface for managing transaction within the runtime. +#[runtime_interface] +pub trait Transaction { + /// Start a new transaction. + fn start_transaction(&mut self) { + self.storage_start_transaction(); + } + + /// Discard a transactional layer. + fn discard_transaction(&mut self) { + self.storage_discard_transaction(); + } + + /// Commit a transactional layer. + fn commit_transaction(&mut self) { + self.storage_commit_transaction(); + } +} + /// Interface that provides trie related functionality. #[runtime_interface] pub trait Trie { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 8a96cd53288ac..f4e78b918ec98 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -310,7 +310,6 @@ impl Externalities for BasicExternalities { fn storage_commit_transaction(&mut self) { // no need to fail in this case } - } impl sp_externalities::ExtensionStore for BasicExternalities { diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index f267f303c715e..bbba263bd6724 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -1099,4 +1099,24 @@ mod tests { runtime_api.test_storage(&block_id).unwrap(); } + + #[test] + fn use_history_data() { + let client = TestClientBuilder::new() + .set_execution_strategy(ExecutionStrategy::AlwaysWasm) + .build(); + let runtime_api = client.runtime_api(); + let block_id = BlockId::Number(client.chain_info().best_number); + assert!(runtime_api.use_history_data(&block_id).is_ok()); + } + + #[test] + fn test_transactions() { + let client = TestClientBuilder::new() + .set_execution_strategy(ExecutionStrategy::AlwaysWasm) + .build(); + let runtime_api = client.runtime_api(); + let block_id = BlockId::Number(client.chain_info().best_number); + assert_eq!(runtime_api.use_transactions(&block_id).unwrap(), 1); + } } From 80f3d38fbe19e4daca46c7ee84c120ade4de3036 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 15:50:42 +0100 Subject: [PATCH 101/129] synch_linear_transaction renamed to sync_linear_transaction --- primitives/historical-data/src/lib.rs | 2 +- ...synch_linear_transaction.rs => sync_linear_transaction.rs} | 0 primitives/state-machine/src/changes_trie/build.rs | 2 +- primitives/state-machine/src/ext.rs | 2 +- primitives/state-machine/src/overlayed_changes.rs | 4 ++-- test-utils/runtime/src/lib.rs | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) rename primitives/historical-data/src/{synch_linear_transaction.rs => sync_linear_transaction.rs} (100%) diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index ec0c6e813fe14..cc5ca38d00898 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -19,7 +19,7 @@ #![cfg_attr(not(feature = "std"), no_std)] -pub mod synch_linear_transaction; +pub mod sync_linear_transaction; /// History of values being used to keep trace of all changes /// that occurs (all different state a value can be in depending diff --git a/primitives/historical-data/src/synch_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs similarity index 100% rename from primitives/historical-data/src/synch_linear_transaction.rs rename to primitives/historical-data/src/sync_linear_transaction.rs diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index eb4919333c8fd..e9423638188d0 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -329,7 +329,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use sp_historical_data::synch_linear_transaction::{States, State, History, HistoricalEntry}; + use sp_historical_data::sync_linear_transaction::{States, State, History, HistoricalEntry}; use super::*; const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 0f89863df7012..640581bf4c14f 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -659,7 +659,7 @@ mod tests { }, backend::InMemory, overlayed_changes::{OverlayedValue, OverlayedChangeSet}, }; use sp_core::storage::{Storage, StorageChild}; - use sp_historical_data::synch_linear_transaction::{History, HistoricalEntry, State}; + use sp_historical_data::sync_linear_transaction::{History, HistoricalEntry, State}; type TestBackend = InMemory; type TestChangesTrieStorage = InMemoryChangesTrieStorage; diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index ab2dde85dc5f5..8382520710702 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -22,7 +22,7 @@ use std::collections::{HashMap, BTreeMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; -use sp_historical_data::synch_linear_transaction::{ +use sp_historical_data::sync_linear_transaction::{ History, HistoricalEntry, States, }; use sp_historical_data::CleaningResult; @@ -74,7 +74,7 @@ pub struct OverlayedChangeSet { #[cfg(test)] impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { - use sp_historical_data::synch_linear_transaction::State; + use sp_historical_data::sync_linear_transaction::State; let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = History::default(); diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index bbba263bd6724..cff1c91ce064c 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -915,8 +915,8 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { } fn test_historical_data() { - let mut states = sp_historical_data::synch_linear_transaction::States::default(); - let mut value = sp_historical_data::synch_linear_transaction::History::default(); + let mut states = sp_historical_data::sync_linear_transaction::States::default(); + let mut value = sp_historical_data::sync_linear_transaction::History::default(); if value.get() != None { panic!("Got a value for empty data"); } From c9873b8678514e9d1c7f7e4522fc0b96bd341d84 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 15:53:26 +0100 Subject: [PATCH 102/129] unused dependency to sp-std --- Cargo.lock | 1 - primitives/historical-data/Cargo.toml | 2 -- 2 files changed, 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f6238c70cb3d..e187aefbc83c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6342,7 +6342,6 @@ name = "sp-historical-data" version = "2.0.0" dependencies = [ "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-std 2.0.0", ] [[package]] diff --git a/primitives/historical-data/Cargo.toml b/primitives/historical-data/Cargo.toml index f4a2a91e4fa3e..5b987e8296fdb 100644 --- a/primitives/historical-data/Cargo.toml +++ b/primitives/historical-data/Cargo.toml @@ -6,13 +6,11 @@ description = "Data associated with its history" edition = "2018" [dependencies] -rstd = { package = "sp-std", path = "../std", default-features = false } smallvec = { version = "0.6", default-features = false } [features] default = ["std"] std = [ - "rstd/std", "smallvec/std", ] test-helpers = [] From 4f0c633b845a4e3822c80139b1e6eac99be42844 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 16:13:39 +0100 Subject: [PATCH 103/129] nits. --- primitives/historical-data/src/lib.rs | 29 ++++++++++--------- .../src/sync_linear_transaction.rs | 19 ++++++------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index cc5ca38d00898..f9b68f20d3260 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -61,30 +61,40 @@ impl HistoricalEntry { } } -#[derive(Debug, PartialEq)] /// The results from changing a historical value. /// It should be used to apply subsequent update the calling context. /// For instance if the historical value was stored into a hashmap, /// then it should be removed from it on a `Cleared` result. +#[derive(Debug, PartialEq)] pub enum CleaningResult { /// No inner data was changed, even technical /// data, therefore no update is needed. Unchanged, - /// Any data got modified, therefore an - /// update may be needed. + /// Historical representation did change. + /// This includes cleaning of deprecated historic values. Changed, /// No historical data is stored anymore, it can be dropped. Cleared, } impl History { - #[cfg(any(test, feature = "test-helpers"))] + /// Push a value without checking if it can overwrite the current + /// state. + /// This should only use after checking the state is correct + /// (last item of historical value got a smaller index than + /// the new one). + pub fn push_unchecked(&mut self, item: HistoricalEntry) { + self.0.push(item); + } +} + +#[cfg(any(test, feature = "test-helpers"))] +impl History { /// Get current number of stored historical values. pub fn len(&self) -> usize { self.0.len() } - #[cfg(any(test, feature = "test-helpers"))] /// Create an history from a sequence of historical values. pub fn from_iter(input: impl IntoIterator>) -> Self { let mut history = History::default(); @@ -93,13 +103,4 @@ impl History { } history } - - /// Push a value without checking if it can overwrite the current - /// state. - /// This should only use after checking the state is correct - /// (last item of historical value got a smaller index than - /// the new one). - pub fn push_unchecked(&mut self, item: HistoricalEntry) { - self.0.push(item); - } } diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs index d61797bafeda0..287c225d781b0 100644 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ b/primitives/historical-data/src/sync_linear_transaction.rs @@ -56,7 +56,6 @@ impl Default for States { } impl States { - /// Get corresponding current state. pub fn as_state(&self) -> State { State::Transaction(self.current_layer) @@ -70,7 +69,7 @@ impl States { States { current_layer } } - /// Update states when discarding prospective changes. + /// Update global state for discarding prospective. /// A subsequent update of all related stored history is needed. pub fn discard_prospective(&mut self) { self.current_layer = 1; @@ -88,21 +87,23 @@ impl States { /// After a prospective was discarded, clear prospective history. pub fn apply_discard_prospective(value: &mut History) -> CleaningResult { - if value.0.len() == 0 { + if value.0.is_empty() { return CleaningResult::Cleared; } if value.0[0].index == State::Committed { if value.0.len() == 1 { - return CleaningResult::Unchanged; + CleaningResult::Unchanged + } else { + value.0.truncate(1); + CleaningResult::Changed } - value.0.truncate(1); } else { value.0.clear(); + CleaningResult::Cleared } - CleaningResult::Changed } - /// Update states when committing prospective. + /// Update global state for committing prospective. /// A subsequent update of all related stored history is needed. pub fn commit_prospective(&mut self) { self.current_layer = 1; @@ -112,7 +113,7 @@ impl States { /// on all values from this prospective. It commits pending value and /// clear existing history. pub fn apply_commit_prospective(value: &mut History) -> CleaningResult { - if value.0.len() == 0 { + if value.0.is_empty() { return CleaningResult::Cleared; } if value.0.len() == 1 { @@ -156,7 +157,7 @@ impl States { } else { break } } else { break } } - if value.0.len() == 0 { + if value.0.is_empty() { CleaningResult::Cleared } else if value.0.len() != init_len { CleaningResult::Changed From b3014ae8c4cc5a2b5d5f725e65a61289a56151a8 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 16:55:42 +0100 Subject: [PATCH 104/129] Comments for `apply_commit_transaction` internals. --- .../src/sync_linear_transaction.rs | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs index 287c225d781b0..524d38bd93b42 100644 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ b/primitives/historical-data/src/sync_linear_transaction.rs @@ -138,9 +138,7 @@ impl States { /// Discard a transactional layer. /// A subsequent synchronous update of all related stored history is needed. pub fn discard_transaction(&mut self) { - if self.current_layer > 0 { - self.current_layer -= 1; - } + self.current_layer = self.current_layer.saturating_sub(1); } /// Apply transaction discard on a historical value. @@ -153,9 +151,13 @@ impl States { index: State::Transaction(ix), } = value.0[i] { if ix > self.current_layer { - let _ = value.0.pop(); - } else { break } - } else { break } + value.0.pop(); + } else { + break; + } + } else { + break; + } } if value.0.is_empty() { CleaningResult::Cleared @@ -179,21 +181,29 @@ impl States { /// Committing value removes all unneeded states. pub fn apply_commit_transaction(&self, value: &mut History) -> CleaningResult { let mut new_value = None; + // Iterate on history to get current committed value and remove + // unused transaction values that are between this new committed + // value and the current transaction layer. for i in (0 .. value.0.len()).rev() { - if let HistoricalEntry { - value: _, - index: State::Transaction(ix), - } = value.0[i] { + if let State::Transaction(ix) = value.0[i].index { if ix > self.current_layer { + // Remove value from committed layer if let Some(v) = value.0.pop() { if new_value.is_none() { new_value = Some(v.value); } } } else if ix == self.current_layer && new_value.is_some() { - let _ = value.0.pop(); - } else { break } - } else { break } + // Remove parent layer value (will be overwritten by `new_value`. + value.0.pop(); + } else { + // Do not remove this is already a valid state. + break; + } + } else { + // Non transactional layer, stop removing history. + break; + } } if let Some(new_value) = new_value { value.0.push(HistoricalEntry { From 412fba46e091a2387abd99bab2e3dd356e54b666 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 17:05:08 +0100 Subject: [PATCH 105/129] Use 'COMMITTED_LAYER' const to make code clearer. --- .../src/sync_linear_transaction.rs | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs index 524d38bd93b42..8ec06da717270 100644 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ b/primitives/historical-data/src/sync_linear_transaction.rs @@ -44,14 +44,17 @@ pub type History = crate::History; /// An entry at a given history height. pub type HistoricalEntry = crate::HistoricalEntry; + +// We use to 1 to be able to discard this transaction. +const COMMITTED_LAYER: usize = 1; + /// Global state contains only the current number of overlay layers. #[derive(Debug, Clone, PartialEq, Eq)] pub struct States { current_layer: usize } impl Default for States { fn default() -> Self { - // we default to 1 to be able to discard this transaction. - States { current_layer: 1 } + States { current_layer: COMMITTED_LAYER } } } @@ -72,17 +75,8 @@ impl States { /// Update global state for discarding prospective. /// A subsequent update of all related stored history is needed. pub fn discard_prospective(&mut self) { - self.current_layer = 1; - } - - /// By default the current transaction is at least 1. - /// 0 is a transient state used when applying transaction change, - /// in this case transaction need to be restored to a valid state afterward. - /// This function does restore the state. - pub fn finalize_discard(&mut self) { - if self.current_layer == 0 { - self.current_layer = 1; - } + // Point to committed layer. + self.current_layer = COMMITTED_LAYER; } /// After a prospective was discarded, clear prospective history. @@ -91,10 +85,10 @@ impl States { return CleaningResult::Cleared; } if value.0[0].index == State::Committed { - if value.0.len() == 1 { + if value.0.len() == COMMITTED_LAYER { CleaningResult::Unchanged } else { - value.0.truncate(1); + value.0.truncate(COMMITTED_LAYER); CleaningResult::Changed } } else { @@ -106,7 +100,8 @@ impl States { /// Update global state for committing prospective. /// A subsequent update of all related stored history is needed. pub fn commit_prospective(&mut self) { - self.current_layer = 1; + // Point to committed layer. + self.current_layer = COMMITTED_LAYER; } /// After updating states to prospective, this function must be use @@ -116,7 +111,7 @@ impl States { if value.0.is_empty() { return CleaningResult::Cleared; } - if value.0.len() == 1 { + if value.0.len() == COMMITTED_LAYER { if value.0[0].index != State::Committed { value.0[0].index = State::Committed; } else { @@ -168,12 +163,21 @@ impl States { } } + /// By default the current transaction is at least 1. + /// 0 is a transient state used when applying transaction change, + /// in this case transaction need to be restored to a valid state afterward. + /// This function does restore the state. + pub fn finalize_discard(&mut self) { + if self.current_layer == 0 { + // Point back to committed layer. + self.current_layer = COMMITTED_LAYER; + } + } + /// Commit a transactional layer. /// A subsequent update of all related stored history is needed. pub fn commit_transaction(&mut self) { - if self.current_layer > 1 { - self.current_layer -= 1; - } + self.current_layer = self.current_layer.saturating_sub(1); } /// Apply transaction commit on a historical value. @@ -313,7 +317,7 @@ impl History { /// Extracts the committed value if there is one. pub fn into_committed(mut self) -> Option { - self.0.truncate(1); + self.0.truncate(COMMITTED_LAYER); if let Some(HistoricalEntry { value, index: State::Committed, From 4f4fed690502b6c8b1a31959a61f6e766e31e7fb Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 17:09:08 +0100 Subject: [PATCH 106/129] Fix refacto error. --- primitives/historical-data/src/sync_linear_transaction.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs index 8ec06da717270..aaf5d2281f813 100644 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ b/primitives/historical-data/src/sync_linear_transaction.rs @@ -177,7 +177,9 @@ impl States { /// Commit a transactional layer. /// A subsequent update of all related stored history is needed. pub fn commit_transaction(&mut self) { - self.current_layer = self.current_layer.saturating_sub(1); + if self.current_layer > COMMITTED_LAYER { + self.current_layer -= 1; + } } /// Apply transaction commit on a historical value. From 9dee5232f422b0c37a4cf6d5c2599088ca98c125 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 17:17:16 +0100 Subject: [PATCH 107/129] Discard transaction is same as truncate. --- .../src/sync_linear_transaction.rs | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs index aaf5d2281f813..4fa95d8c09813 100644 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ b/primitives/historical-data/src/sync_linear_transaction.rs @@ -140,20 +140,8 @@ impl States { /// Multiple calls to `discard_transaction` can be applied at once. pub fn apply_discard_transaction(&self, value: &mut History) -> CleaningResult { let init_len = value.0.len(); - for i in (0 .. value.0.len()).rev() { - if let HistoricalEntry { - value: _, - index: State::Transaction(ix), - } = value.0[i] { - if ix > self.current_layer { - value.0.pop(); - } else { - break; - } - } else { - break; - } - } + // Remove all transactional layers. + value.0.truncate(self.current_layer); if value.0.is_empty() { CleaningResult::Cleared } else if value.0.len() != init_len { From f055f3bff12cef33ae787e77f044322eb101ba82 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 7 Jan 2020 17:23:42 +0100 Subject: [PATCH 108/129] Move helpers implementations. --- .../src/sync_linear_transaction.rs | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs index 4fa95d8c09813..46ee177deffe3 100644 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ b/primitives/historical-data/src/sync_linear_transaction.rs @@ -233,7 +233,6 @@ impl State { } impl History { - /// Set a value, this use a global state as parameter. pub fn set(&mut self, states: &States, value: V) { if let Some(v) = self.0.last_mut() { @@ -266,9 +265,29 @@ impl History { } } + /// Extracts the committed value if there is one. + pub fn into_committed(mut self) -> Option { + self.0.truncate(COMMITTED_LAYER); + if let Some(HistoricalEntry { + value, + index: State::Committed, + }) = self.0.pop() { + return Some(value) + } else { + None + } + } + + /// Returns mutable handle on latest pending historical value. + pub fn get_mut(&mut self) -> Option> { + self.0.last_mut().map(|h| h.as_mut()) + } +} + +#[cfg(any(test, feature = "test-helpers"))] +impl History { /// Get latest prospective value, excludes /// committed values. - #[cfg(any(test, feature = "test-helpers"))] pub fn get_prospective(&self) -> Option<&V> { match self.0.get(0) { Some(HistoricalEntry { @@ -293,7 +312,6 @@ impl History { } /// Get latest committed value. - #[cfg(any(test, feature = "test-helpers"))] pub fn get_committed(&self) -> Option<&V> { if let Some(HistoricalEntry { value, @@ -304,22 +322,4 @@ impl History { None } } - - /// Extracts the committed value if there is one. - pub fn into_committed(mut self) -> Option { - self.0.truncate(COMMITTED_LAYER); - if let Some(HistoricalEntry { - value, - index: State::Committed, - }) = self.0.pop() { - return Some(value) - } else { - None - } - } - - /// Returns mutable handle on latest pending historical value. - pub fn get_mut(&mut self) -> Option> { - self.0.last_mut().map(|h| h.as_mut()) - } } From 441d5ca16696913fdab841b3196af57e39ae1f10 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 8 Jan 2020 10:39:28 +0100 Subject: [PATCH 109/129] Declare transaction host function for wasm --- primitives/io/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 5459a35990f8e..74bb3acda9759 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -908,6 +908,7 @@ pub type TestExternalities = sp_state_machine::TestExternalities Date: Fri, 10 Jan 2020 09:54:05 +0100 Subject: [PATCH 110/129] renaming prior to removal of historical-data --- primitives/historical-data/src/lib.rs | 44 +++-- .../src/sync_linear_transaction.rs | 162 +++++++----------- .../state-machine/src/overlayed_changes.rs | 78 ++++----- test-utils/runtime/src/lib.rs | 26 --- 4 files changed, 118 insertions(+), 192 deletions(-) diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs index f9b68f20d3260..3203bd51b855c 100644 --- a/primitives/historical-data/src/lib.rs +++ b/primitives/historical-data/src/lib.rs @@ -21,43 +21,39 @@ pub mod sync_linear_transaction; -/// History of values being used to keep trace of all changes -/// that occurs (all different state a value can be in depending -/// on the global state). -/// The different states of this value, are ordered by change time -/// in a simple stack. +/// Stack of values in different transactional layers. #[derive(Debug, Clone, PartialEq)] -pub struct History(pub(crate) smallvec::SmallVec<[HistoricalEntry; ALLOCATED_HISTORY]>); +pub struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); /// Size of preallocated history per element. /// Current size is two, that is currently related to the use case /// where value got `committed` and `prospective` initial state by default. const ALLOCATED_HISTORY: usize = 2; -impl Default for History { +impl Default for Layers { fn default() -> Self { - History(Default::default()) + Layers(Default::default()) } } -/// An entry of a historical data at a given history point. +/// An value entry in a given indexed transactional layer. #[derive(Debug, Clone, PartialEq)] -pub struct HistoricalEntry { +pub struct LayerEntry { /// The stored value. pub value: V, - /// The moment in history when the value got set. - pub index: I, + /// The transactional layer index. + pub index: usize, } -impl From<(V, I)> for HistoricalEntry { - fn from(input: (V, I)) -> HistoricalEntry { - HistoricalEntry { value: input.0, index: input.1 } +impl From<(V, usize)> for LayerEntry { + fn from(input: (V, usize)) -> LayerEntry { + LayerEntry { value: input.0, index: input.1 } } } -impl HistoricalEntry { - fn as_mut(&mut self) -> HistoricalEntry<&mut V, I> { - HistoricalEntry { value: &mut self.value, index: self.index.clone() } +impl LayerEntry { + fn as_mut(&mut self) -> LayerEntry<&mut V> { + LayerEntry { value: &mut self.value, index: self.index } } } @@ -66,7 +62,7 @@ impl HistoricalEntry { /// For instance if the historical value was stored into a hashmap, /// then it should be removed from it on a `Cleared` result. #[derive(Debug, PartialEq)] -pub enum CleaningResult { +pub enum LayeredOpsResult { /// No inner data was changed, even technical /// data, therefore no update is needed. Unchanged, @@ -77,27 +73,27 @@ pub enum CleaningResult { Cleared, } -impl History { +impl Layers { /// Push a value without checking if it can overwrite the current /// state. /// This should only use after checking the state is correct /// (last item of historical value got a smaller index than /// the new one). - pub fn push_unchecked(&mut self, item: HistoricalEntry) { + pub fn push_unchecked(&mut self, item: LayerEntry) { self.0.push(item); } } #[cfg(any(test, feature = "test-helpers"))] -impl History { +impl Layers { /// Get current number of stored historical values. pub fn len(&self) -> usize { self.0.len() } /// Create an history from a sequence of historical values. - pub fn from_iter(input: impl IntoIterator>) -> Self { - let mut history = History::default(); + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = Layers::default(); for v in input { history.push_unchecked(v); } diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs index 46ee177deffe3..cd40059c40bad 100644 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ b/primitives/historical-data/src/sync_linear_transaction.rs @@ -36,93 +36,64 @@ //! of creation is also needed. //! Transaction operation does not interfere with value in `Committed` state. -use crate::CleaningResult; +use crate::{LayeredOpsResult, Layers, LayerEntry}; -/// History of value and their state. -pub type History = crate::History; - -/// An entry at a given history height. -pub type HistoricalEntry = crate::HistoricalEntry; - - -// We use to 1 to be able to discard this transaction. -const COMMITTED_LAYER: usize = 1; - -/// Global state contains only the current number of overlay layers. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct States { current_layer: usize } - -impl Default for States { - fn default() -> Self { - States { current_layer: COMMITTED_LAYER } - } -} +/// Index of commited layer, this is reserved. +const COMMITTED_LAYER: usize = 0; +/* impl States { - /// Get corresponding current state. - pub fn as_state(&self) -> State { - State::Transaction(self.current_layer) - } - - /// Instantiate a random state, only for testing. - #[cfg(any(test, feature = "test-helpers"))] - pub fn test_state( - current_layer: usize, - ) -> Self { - States { current_layer } - } - /// Update global state for discarding prospective. /// A subsequent update of all related stored history is needed. pub fn discard_prospective(&mut self) { - // Point to committed layer. - self.current_layer = COMMITTED_LAYER; + // Point to prospective layer. + self.current_layer = COMMITTED_LAYER + 1; } /// After a prospective was discarded, clear prospective history. - pub fn apply_discard_prospective(value: &mut History) -> CleaningResult { + pub fn apply_discard_prospective(value: &mut Layers) -> LayeredOpsResult { if value.0.is_empty() { - return CleaningResult::Cleared; + return LayeredOpsResult::Cleared; } if value.0[0].index == State::Committed { - if value.0.len() == COMMITTED_LAYER { - CleaningResult::Unchanged + if value.0.len() == COMMITTED_LAYER + 1 { + LayeredOpsResult::Unchanged } else { - value.0.truncate(COMMITTED_LAYER); - CleaningResult::Changed + value.0.truncate(COMMITTED_LAYER + 1); + LayeredOpsResult::Changed } } else { value.0.clear(); - CleaningResult::Cleared + LayeredOpsResult::Cleared } } /// Update global state for committing prospective. /// A subsequent update of all related stored history is needed. pub fn commit_prospective(&mut self) { - // Point to committed layer. - self.current_layer = COMMITTED_LAYER; + // Point to prospective layer. + self.current_layer = COMMITTED_LAYER + 1; } /// After updating states to prospective, this function must be use /// on all values from this prospective. It commits pending value and /// clear existing history. - pub fn apply_commit_prospective(value: &mut History) -> CleaningResult { + pub fn apply_commit_prospective(value: &mut Layers) -> LayeredOpsResult { if value.0.is_empty() { - return CleaningResult::Cleared; + return LayeredOpsResult::Cleared; } - if value.0.len() == COMMITTED_LAYER { + if value.0.len() == COMMITTED_LAYER + 1 { if value.0[0].index != State::Committed { value.0[0].index = State::Committed; } else { - return CleaningResult::Unchanged; + return LayeredOpsResult::Unchanged; } } else if let Some(mut v) = value.0.pop() { v.index = State::Committed; value.0.clear(); value.0.push(v); } - CleaningResult::Changed + LayeredOpsResult::Changed } /// Create a new transactional layer. @@ -138,16 +109,16 @@ impl States { /// Apply transaction discard on a historical value. /// Multiple calls to `discard_transaction` can be applied at once. - pub fn apply_discard_transaction(&self, value: &mut History) -> CleaningResult { + pub fn apply_discard_transaction(&self, value: &mut Layers) -> LayeredOpsResult { let init_len = value.0.len(); // Remove all transactional layers. value.0.truncate(self.current_layer); if value.0.is_empty() { - CleaningResult::Cleared + LayeredOpsResult::Cleared } else if value.0.len() != init_len { - CleaningResult::Changed + LayeredOpsResult::Changed } else { - CleaningResult::Unchanged + LayeredOpsResult::Unchanged } } @@ -158,14 +129,14 @@ impl States { pub fn finalize_discard(&mut self) { if self.current_layer == 0 { // Point back to committed layer. - self.current_layer = COMMITTED_LAYER; + self.current_layer = COMMITTED_LAYER + 1; } } /// Commit a transactional layer. /// A subsequent update of all related stored history is needed. pub fn commit_transaction(&mut self) { - if self.current_layer > COMMITTED_LAYER { + if self.current_layer > COMMITTED_LAYER + 1 { self.current_layer -= 1; } } @@ -173,7 +144,7 @@ impl States { /// Apply transaction commit on a historical value. /// Multiple calls to `commit_transaction` can be applied at once. /// Committing value removes all unneeded states. - pub fn apply_commit_transaction(&self, value: &mut History) -> CleaningResult { + pub fn apply_commit_transaction(&self, value: &mut Layers) -> LayeredOpsResult { let mut new_value = None; // Iterate on history to get current committed value and remove // unused transaction values that are between this new committed @@ -200,28 +171,16 @@ impl States { } } if let Some(new_value) = new_value { - value.0.push(HistoricalEntry { + value.0.push(LayerEntry { value: new_value, index: State::Transaction(self.current_layer), }); - return CleaningResult::Changed; + return LayeredOpsResult::Changed; } - CleaningResult::Unchanged + LayeredOpsResult::Unchanged } } -/// Historical value state of multiple transaction -/// with an alterantive committed state. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum State { - /// Committed, in this state no action on transaction - /// can modify this value. - Committed, - /// Value relates to a transaction, the state contains the current - /// number of transaction when value did change. - Transaction(usize), -} - impl State { fn transaction_index(&self) -> Option { if let &State::Transaction(ix) = self { @@ -231,23 +190,24 @@ impl State { } } } +*/ -impl History { +impl Layers { /// Set a value, this use a global state as parameter. - pub fn set(&mut self, states: &States, value: V) { + pub fn set(&mut self, number_transactions: usize, value: V) { if let Some(v) = self.0.last_mut() { - debug_assert!(v.index.transaction_index().unwrap_or(0) <= states.current_layer, - "History expects \ - only new values at the latest state, some state has not \ - synchronized properly"); - if v.index.transaction_index() == Some(states.current_layer) { + debug_assert!(v.index <= number_transactions, + "Layers expects \ + only new values at the latest transaction \ + this indicates some unsynchronized layer value."); + if v.index == number_transactions { v.value = value; return; } } - self.0.push(HistoricalEntry { + self.0.push(LayerEntry { value, - index: State::Transaction(states.current_layer), + index: number_transactions, }); } @@ -267,10 +227,10 @@ impl History { /// Extracts the committed value if there is one. pub fn into_committed(mut self) -> Option { - self.0.truncate(COMMITTED_LAYER); - if let Some(HistoricalEntry { + self.0.truncate(COMMITTED_LAYER + 1); + if let Some(LayerEntry { value, - index: State::Committed, + index: COMMITTED_LAYER, }) = self.0.pop() { return Some(value) } else { @@ -279,45 +239,41 @@ impl History { } /// Returns mutable handle on latest pending historical value. - pub fn get_mut(&mut self) -> Option> { + pub fn get_mut(&mut self) -> Option> { self.0.last_mut().map(|h| h.as_mut()) } } #[cfg(any(test, feature = "test-helpers"))] -impl History { +impl Layers { /// Get latest prospective value, excludes /// committed values. pub fn get_prospective(&self) -> Option<&V> { match self.0.get(0) { - Some(HistoricalEntry { - value: _, - index: State::Committed, - }) => { - if let Some(HistoricalEntry { - value, - index: State::Transaction(_), - }) = self.0.get(1) { - Some(&value) + Some(entry) if entry.index == COMMITTED_LAYER => { + if let Some(entry) = self.0.get(1) { + if entry.index > COMMITTED_LAYER { + Some(&entry.value) + } else { + None + } } else { None } }, - Some(HistoricalEntry { - value, - index: State::Transaction(_), - }) => Some(&value), + Some(entry) => Some(&entry.value), None => None, } } /// Get latest committed value. pub fn get_committed(&self) -> Option<&V> { - if let Some(HistoricalEntry { - value, - index: State::Committed, - }) = self.0.get(0) { - return Some(&value) + if let Some(entry) = self.0.get(0) { + if entry.index == COMMITTED_LAYER { + return Some(&entry.value) + } else { + None + } } else { None } diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 8382520710702..25acf93b25d40 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -23,9 +23,9 @@ use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; use sp_historical_data::sync_linear_transaction::{ - History, HistoricalEntry, States, + Layers, LayerEntry, }; -use sp_historical_data::CleaningResult; +use sp_historical_data::LayeredOpsResult; use std::ops; /// The overlayed changes to state to be queried on top of the backend. @@ -52,7 +52,7 @@ pub struct OverlayedValue { pub extrinsics: Option>, } -type TreeChangeSet = BTreeMap, History>; +type TreeChangeSet = BTreeMap, Layers>; /// Overlayed change set, content keep trace of its history. /// @@ -61,13 +61,13 @@ type TreeChangeSet = BTreeMap, History>; #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { /// Indexed state history. - pub(crate) states: States, + pub(crate) number_transaction: usize, /// Top level storage changes. pub(crate) top: TreeChangeSet, /// Child storage changes. /// OwnedChildInfo is currently an absolute value, for some child trie /// operations (eg full deletion) it will need to change - /// to `History`. + /// to `Layers`. pub(crate) children: HashMap, (TreeChangeSet, OwnedChildInfo)>, } @@ -77,8 +77,8 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { use sp_historical_data::sync_linear_transaction::State; let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { - let mut history = History::default(); - history.push_unchecked(HistoricalEntry { value, index: State::Committed }); + let mut history = Layers::default(); + history.push_unchecked(LayerEntry { value, index: State::Committed }); history })).collect(); result @@ -89,15 +89,15 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { /// It avoids accessing two times the historical value item. /// It does remove latest historical dropped items. fn set_with_extrinsic_overlayed_value( - history: &mut History, - states: &States, + history: &mut Layers, + number_transaction: usize, value: Option>, extrinsic_index: Option, ) { if let Some(extrinsic) = extrinsic_index { - set_with_extrinsic_inner_overlayed_value(history, states, value, extrinsic) + set_with_extrinsic_inner_overlayed_value(history, number_transaction, value, extrinsic) } else { - history.set(states, OverlayedValue { + history.set(number_transaction, OverlayedValue { value, extrinsics: None, }) @@ -105,12 +105,12 @@ fn set_with_extrinsic_overlayed_value( } fn set_with_extrinsic_inner_overlayed_value( - history: &mut History, - states: &States, + history: &mut Layers, + number_transaction: usize, value: Option>, extrinsic_index: u32, ) { - let state = states.as_state(); + let state = number_transaction.as_state(); if let Some(current) = history.get_mut() { if current.index == state { current.value.value = value; @@ -120,7 +120,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics = current.value.extrinsics.clone(); extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - history.push_unchecked(HistoricalEntry { + history.push_unchecked(LayerEntry { index: state, value: OverlayedValue { value, @@ -132,7 +132,7 @@ fn set_with_extrinsic_inner_overlayed_value( let mut extrinsics: Option> = None; extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); - history.push_unchecked(HistoricalEntry { + history.push_unchecked(LayerEntry { index: state, value: OverlayedValue { value, @@ -150,50 +150,50 @@ impl OverlayedChangeSet { /// Discard prospective changes from the change set. pub fn discard_prospective(&mut self) { - self.states.discard_prospective(); - retain(&mut self.top, |_, history| States::apply_discard_prospective(history) != CleaningResult::Cleared); + self.number_transaction.discard_prospective(); + retain(&mut self.top, |_, history| States::apply_discard_prospective(history) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| States::apply_discard_prospective(history) != CleaningResult::Cleared); + retain(map, |_, history| States::apply_discard_prospective(history) != LayeredOpsResult::Cleared); !map.is_empty() }); } /// Commit prospective changes into the change set. pub fn commit_prospective(&mut self) { - self.states.commit_prospective(); - retain(&mut self.top, |_, history| States::apply_commit_prospective(history) != CleaningResult::Cleared); + self.number_transaction.commit_prospective(); + retain(&mut self.top, |_, history| States::apply_commit_prospective(history) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| States::apply_commit_prospective(history) != CleaningResult::Cleared); + retain(map, |_, history| States::apply_commit_prospective(history) != LayeredOpsResult::Cleared); !map.is_empty() }); } /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.states.start_transaction(); + self.number_transaction.start_transaction(); } /// Discard a transactional layer. /// There is always a transactional layer running /// (discarding the last trasactional layer open a new one). pub fn discard_transaction(&mut self) { - self.states.discard_transaction(); - let states = &self.states; - retain(&mut self.top, |_, history| states.apply_discard_transaction(history) != CleaningResult::Cleared); + self.number_transaction.discard_transaction(); + let number_transaction = &self.number_transaction; + retain(&mut self.top, |_, history| number_transaction.apply_discard_transaction(history) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| states.apply_discard_transaction(history) != CleaningResult::Cleared); + retain(map, |_, history| number_transaction.apply_discard_transaction(history) != LayeredOpsResult::Cleared); !map.is_empty() }); - self.states.finalize_discard(); + self.number_transaction.finalize_discard(); } /// Commit a transactional layer into previous transaction layer. pub fn commit_transaction(&mut self) { - self.states.commit_transaction(); - let states = &self.states; - retain(&mut self.top, |_, history| states.apply_commit_transaction(history) != CleaningResult::Cleared); + self.number_transaction.commit_transaction(); + let number_transaction = &self.number_transaction; + retain(&mut self.top, |_, history| number_transaction.apply_commit_transaction(history) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| states.apply_commit_transaction(history) != CleaningResult::Cleared); + retain(map, |_, history| number_transaction.apply_commit_transaction(history) != LayeredOpsResult::Cleared); !map.is_empty() }); } @@ -372,7 +372,7 @@ impl OverlayedChanges { pub fn set_storage(&mut self, key: Vec, value: Option>) { let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - set_with_extrinsic_overlayed_value(entry, &self.changes.states, value, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, &self.changes.number_transaction, value, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -394,7 +394,7 @@ impl OverlayedChanges { let entry = map_entry.0.entry(key).or_default(); set_with_extrinsic_overlayed_value( entry, - &self.changes.states, + &self.changes.number_transaction, value, extrinsic_index, ); @@ -412,14 +412,14 @@ impl OverlayedChanges { child_info: ChildInfo, ) { let extrinsic_index = self.extrinsic_index(); - let states = &self.changes.states; + let number_transaction = &self.changes.number_transaction; let map_entry = self.changes.children.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); let updatable = map_entry.1.try_update(child_info); debug_assert!(updatable); map_entry.0.values_mut() - .for_each(|e| set_with_extrinsic_overlayed_value(e, states, None, extrinsic_index)); + .for_each(|e| set_with_extrinsic_overlayed_value(e, number_transaction, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -433,7 +433,7 @@ impl OverlayedChanges { for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, &self.changes.number_transaction, None, extrinsic_index); } } } @@ -451,7 +451,7 @@ impl OverlayedChanges { for (key, entry) in child_change.0.iter_mut() { if key.starts_with(prefix) { - set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, &self.changes.number_transaction, None, extrinsic_index); } } } @@ -556,7 +556,7 @@ impl OverlayedChanges { } #[cfg(any(test, feature = "test-helpers"))] - /// Count the number of key value pairs, at every history states. + /// Count the number of key value pairs, at every history number_transaction. /// Should only be use for debugging or testing, this is slow /// and technical. pub fn top_count_keyvalue_pair(&self) -> usize { diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index cff1c91ce064c..e2c5f8d161dbe 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -255,8 +255,6 @@ cfg_if! { fn fail_on_wasm() -> u64; /// Trie no_std testing. fn use_trie() -> u64; - /// History data no_std testing. - fn use_history_data(); /// Transactional tests. fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; @@ -914,30 +912,6 @@ fn test_sr25519_crypto() -> (sr25519::AppSignature, sr25519::AppPublic) { (signature, public0) } -fn test_historical_data() { - let mut states = sp_historical_data::sync_linear_transaction::States::default(); - let mut value = sp_historical_data::sync_linear_transaction::History::default(); - if value.get() != None { - panic!("Got a value for empty data"); - } - - value.set(&states, 42u64); - states.start_transaction(); - if value.get() != Some(&42) { - panic!("Got a wrong result accessing a one element data"); - } - value.set(&states, 43u64); - if value.get() != Some(&43) { - panic!("Got a wrong result accessing a two element data"); - } - states.discard_transaction(); - states.apply_discard_transaction(&mut value); - states.finalize_discard(); - if value.get() != Some(&42) { - panic!("Got a wrong result accessing a one element data after a discard"); - } -} - fn test_read_storage() { const KEY: &[u8] = b":read_storage"; sp_io::storage::set(KEY, b"test"); From b3078a7fe0269a9d965621b9f0627d9eaef18447 Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 10 Jan 2020 14:21:03 +0100 Subject: [PATCH 111/129] Flattern historical data crate & fix discard transaction latest commit regression. --- Cargo.lock | 10 +- Cargo.toml | 1 - primitives/historical-data/Cargo.toml | 16 - primitives/historical-data/README.md | 20 -- primitives/historical-data/src/lib.rs | 102 ------- .../src/sync_linear_transaction.rs | 281 ------------------ primitives/state-machine/Cargo.toml | 7 +- primitives/state-machine/fuzz/Cargo.lock | 63 ++-- primitives/state-machine/fuzz/src/lib.rs | 5 +- .../state-machine/src/changes_trie/build.rs | 50 ++-- primitives/state-machine/src/ext.rs | 17 +- primitives/state-machine/src/lib.rs | 249 ++++++++++++++++ .../state-machine/src/overlayed_changes.rs | 80 ++--- test-utils/runtime/Cargo.toml | 2 - test-utils/runtime/src/lib.rs | 20 -- 15 files changed, 356 insertions(+), 567 deletions(-) delete mode 100644 primitives/historical-data/Cargo.toml delete mode 100644 primitives/historical-data/README.md delete mode 100644 primitives/historical-data/src/lib.rs delete mode 100644 primitives/historical-data/src/sync_linear_transaction.rs diff --git a/Cargo.lock b/Cargo.lock index e187aefbc83c2..d6b54721f01d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6337,13 +6337,6 @@ dependencies = [ "sp-std 2.0.0", ] -[[package]] -name = "sp-historical-data" -version = "2.0.0" -dependencies = [ - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "sp-inherents" version = "2.0.0" @@ -6542,9 +6535,9 @@ dependencies = [ "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", "sp-core 2.0.0", "sp-externalities 2.0.0", - "sp-historical-data 2.0.0", "sp-panic-handler 2.0.0", "sp-trie 2.0.0", "trie-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -6844,7 +6837,6 @@ dependencies = [ "sp-consensus-aura 0.8.0", "sp-consensus-babe 0.8.0", "sp-core 2.0.0", - "sp-historical-data 2.0.0", "sp-inherents 2.0.0", "sp-io 2.0.0", "sp-keyring 2.0.0", diff --git a/Cargo.toml b/Cargo.toml index 9fbacdbb50ec1..505cd299d9c71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,7 +128,6 @@ members = [ "primitives/api/proc-macro", "primitives/api/test", "primitives/arithmetic", - "primitives/historical-data", "primitives/io", "primitives/runtime", "primitives/sandbox", diff --git a/primitives/historical-data/Cargo.toml b/primitives/historical-data/Cargo.toml deleted file mode 100644 index 5b987e8296fdb..0000000000000 --- a/primitives/historical-data/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "sp-historical-data" -version = "2.0.0" -authors = ["Parity Technologies "] -description = "Data associated with its history" -edition = "2018" - -[dependencies] -smallvec = { version = "0.6", default-features = false } - -[features] -default = ["std"] -std = [ - "smallvec/std", -] -test-helpers = [] diff --git a/primitives/historical-data/README.md b/primitives/historical-data/README.md deleted file mode 100644 index 4b14e37e1749b..0000000000000 --- a/primitives/historical-data/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## Historical data - -Crate for managing data and their associated history. - -This covers: -- data with a sequential history, eg. transactional layers and their associated changes. - -This crate is storing single historical data, for instance in a sequential list of state. -Historical collection of data will therefore be preferably a collection of individual historical data. -For instance with historical data being a list of change, a collection of historical data will -be a collection of list rather than a list of collection. - -For multiple individual data, we refer to a global historical state, -query and update actions are using it as an input parameter. - -This crate is `no_std` compatible if the `std` feature is not enabled. - -For more information see - -License: GPL-3.0 diff --git a/primitives/historical-data/src/lib.rs b/primitives/historical-data/src/lib.rs deleted file mode 100644 index 3203bd51b855c..0000000000000 --- a/primitives/historical-data/src/lib.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Data storage containing multiple states of a value. -//! This is used to store historical information for an item. - -#![cfg_attr(not(feature = "std"), no_std)] - -pub mod sync_linear_transaction; - -/// Stack of values in different transactional layers. -#[derive(Debug, Clone, PartialEq)] -pub struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); - -/// Size of preallocated history per element. -/// Current size is two, that is currently related to the use case -/// where value got `committed` and `prospective` initial state by default. -const ALLOCATED_HISTORY: usize = 2; - -impl Default for Layers { - fn default() -> Self { - Layers(Default::default()) - } -} - -/// An value entry in a given indexed transactional layer. -#[derive(Debug, Clone, PartialEq)] -pub struct LayerEntry { - /// The stored value. - pub value: V, - /// The transactional layer index. - pub index: usize, -} - -impl From<(V, usize)> for LayerEntry { - fn from(input: (V, usize)) -> LayerEntry { - LayerEntry { value: input.0, index: input.1 } - } -} - -impl LayerEntry { - fn as_mut(&mut self) -> LayerEntry<&mut V> { - LayerEntry { value: &mut self.value, index: self.index } - } -} - -/// The results from changing a historical value. -/// It should be used to apply subsequent update the calling context. -/// For instance if the historical value was stored into a hashmap, -/// then it should be removed from it on a `Cleared` result. -#[derive(Debug, PartialEq)] -pub enum LayeredOpsResult { - /// No inner data was changed, even technical - /// data, therefore no update is needed. - Unchanged, - /// Historical representation did change. - /// This includes cleaning of deprecated historic values. - Changed, - /// No historical data is stored anymore, it can be dropped. - Cleared, -} - -impl Layers { - /// Push a value without checking if it can overwrite the current - /// state. - /// This should only use after checking the state is correct - /// (last item of historical value got a smaller index than - /// the new one). - pub fn push_unchecked(&mut self, item: LayerEntry) { - self.0.push(item); - } -} - -#[cfg(any(test, feature = "test-helpers"))] -impl Layers { - /// Get current number of stored historical values. - pub fn len(&self) -> usize { - self.0.len() - } - - /// Create an history from a sequence of historical values. - pub fn from_iter(input: impl IntoIterator>) -> Self { - let mut history = Layers::default(); - for v in input { - history.push_unchecked(v); - } - history - } -} diff --git a/primitives/historical-data/src/sync_linear_transaction.rs b/primitives/historical-data/src/sync_linear_transaction.rs deleted file mode 100644 index cd40059c40bad..0000000000000 --- a/primitives/historical-data/src/sync_linear_transaction.rs +++ /dev/null @@ -1,281 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Linear sequence of historical data with transactional -//! support. -//! -//! This is mainly synchronous, global transactional state operation -//! do update value synchronously (to gain time on those operation a -//! different design could be to switch to only change global state -//! and apply garbage collection later if needed). -//! -//! # Global state -//! -//! The global state is the current number of overlayed transaction layers. -//! Committing or discarding a layer simply change this counter and a -//! synchronous local state operations should follow to update every -//! associated historical data. -//! -//! # Local state -//! -//! Local state is either defined as `Committed` or `Prospective`. -//! If in `Prospective` state, the index of the number of layesr at the time -//! of creation is also needed. -//! Transaction operation does not interfere with value in `Committed` state. - -use crate::{LayeredOpsResult, Layers, LayerEntry}; - -/// Index of commited layer, this is reserved. -const COMMITTED_LAYER: usize = 0; - -/* -impl States { - /// Update global state for discarding prospective. - /// A subsequent update of all related stored history is needed. - pub fn discard_prospective(&mut self) { - // Point to prospective layer. - self.current_layer = COMMITTED_LAYER + 1; - } - - /// After a prospective was discarded, clear prospective history. - pub fn apply_discard_prospective(value: &mut Layers) -> LayeredOpsResult { - if value.0.is_empty() { - return LayeredOpsResult::Cleared; - } - if value.0[0].index == State::Committed { - if value.0.len() == COMMITTED_LAYER + 1 { - LayeredOpsResult::Unchanged - } else { - value.0.truncate(COMMITTED_LAYER + 1); - LayeredOpsResult::Changed - } - } else { - value.0.clear(); - LayeredOpsResult::Cleared - } - } - - /// Update global state for committing prospective. - /// A subsequent update of all related stored history is needed. - pub fn commit_prospective(&mut self) { - // Point to prospective layer. - self.current_layer = COMMITTED_LAYER + 1; - } - - /// After updating states to prospective, this function must be use - /// on all values from this prospective. It commits pending value and - /// clear existing history. - pub fn apply_commit_prospective(value: &mut Layers) -> LayeredOpsResult { - if value.0.is_empty() { - return LayeredOpsResult::Cleared; - } - if value.0.len() == COMMITTED_LAYER + 1 { - if value.0[0].index != State::Committed { - value.0[0].index = State::Committed; - } else { - return LayeredOpsResult::Unchanged; - } - } else if let Some(mut v) = value.0.pop() { - v.index = State::Committed; - value.0.clear(); - value.0.push(v); - } - LayeredOpsResult::Changed - } - - /// Create a new transactional layer. - pub fn start_transaction(&mut self) { - self.current_layer += 1; - } - - /// Discard a transactional layer. - /// A subsequent synchronous update of all related stored history is needed. - pub fn discard_transaction(&mut self) { - self.current_layer = self.current_layer.saturating_sub(1); - } - - /// Apply transaction discard on a historical value. - /// Multiple calls to `discard_transaction` can be applied at once. - pub fn apply_discard_transaction(&self, value: &mut Layers) -> LayeredOpsResult { - let init_len = value.0.len(); - // Remove all transactional layers. - value.0.truncate(self.current_layer); - if value.0.is_empty() { - LayeredOpsResult::Cleared - } else if value.0.len() != init_len { - LayeredOpsResult::Changed - } else { - LayeredOpsResult::Unchanged - } - } - - /// By default the current transaction is at least 1. - /// 0 is a transient state used when applying transaction change, - /// in this case transaction need to be restored to a valid state afterward. - /// This function does restore the state. - pub fn finalize_discard(&mut self) { - if self.current_layer == 0 { - // Point back to committed layer. - self.current_layer = COMMITTED_LAYER + 1; - } - } - - /// Commit a transactional layer. - /// A subsequent update of all related stored history is needed. - pub fn commit_transaction(&mut self) { - if self.current_layer > COMMITTED_LAYER + 1 { - self.current_layer -= 1; - } - } - - /// Apply transaction commit on a historical value. - /// Multiple calls to `commit_transaction` can be applied at once. - /// Committing value removes all unneeded states. - pub fn apply_commit_transaction(&self, value: &mut Layers) -> LayeredOpsResult { - let mut new_value = None; - // Iterate on history to get current committed value and remove - // unused transaction values that are between this new committed - // value and the current transaction layer. - for i in (0 .. value.0.len()).rev() { - if let State::Transaction(ix) = value.0[i].index { - if ix > self.current_layer { - // Remove value from committed layer - if let Some(v) = value.0.pop() { - if new_value.is_none() { - new_value = Some(v.value); - } - } - } else if ix == self.current_layer && new_value.is_some() { - // Remove parent layer value (will be overwritten by `new_value`. - value.0.pop(); - } else { - // Do not remove this is already a valid state. - break; - } - } else { - // Non transactional layer, stop removing history. - break; - } - } - if let Some(new_value) = new_value { - value.0.push(LayerEntry { - value: new_value, - index: State::Transaction(self.current_layer), - }); - return LayeredOpsResult::Changed; - } - LayeredOpsResult::Unchanged - } -} - -impl State { - fn transaction_index(&self) -> Option { - if let &State::Transaction(ix) = self { - Some(ix) - } else { - None - } - } -} -*/ - -impl Layers { - /// Set a value, this use a global state as parameter. - pub fn set(&mut self, number_transactions: usize, value: V) { - if let Some(v) = self.0.last_mut() { - debug_assert!(v.index <= number_transactions, - "Layers expects \ - only new values at the latest transaction \ - this indicates some unsynchronized layer value."); - if v.index == number_transactions { - v.value = value; - return; - } - } - self.0.push(LayerEntry { - value, - index: number_transactions, - }); - } - - /// Access to the latest pending value. - pub fn get(&self) -> Option<&V> { - self.0.last().map(|h| &h.value) - } - - /// Extracts the latest pending value if there is one. - pub fn into_pending(mut self) -> Option { - if let Some(v) = self.0.pop() { - Some(v.value) - } else { - None - } - } - - /// Extracts the committed value if there is one. - pub fn into_committed(mut self) -> Option { - self.0.truncate(COMMITTED_LAYER + 1); - if let Some(LayerEntry { - value, - index: COMMITTED_LAYER, - }) = self.0.pop() { - return Some(value) - } else { - None - } - } - - /// Returns mutable handle on latest pending historical value. - pub fn get_mut(&mut self) -> Option> { - self.0.last_mut().map(|h| h.as_mut()) - } -} - -#[cfg(any(test, feature = "test-helpers"))] -impl Layers { - /// Get latest prospective value, excludes - /// committed values. - pub fn get_prospective(&self) -> Option<&V> { - match self.0.get(0) { - Some(entry) if entry.index == COMMITTED_LAYER => { - if let Some(entry) = self.0.get(1) { - if entry.index > COMMITTED_LAYER { - Some(&entry.value) - } else { - None - } - } else { - None - } - }, - Some(entry) => Some(&entry.value), - None => None, - } - } - - /// Get latest committed value. - pub fn get_committed(&self) -> Option<&V> { - if let Some(entry) = self.0.get(0) { - if entry.index == COMMITTED_LAYER { - return Some(&entry.value) - } else { - None - } - } else { - None - } - } -} diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 27f3d4edcc0c1..697c805b56d54 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -11,11 +11,11 @@ parking_lot = "0.9.0" hash-db = "0.15.2" trie-db = "0.18.1" trie-root = "0.15.2" +smallvec = "0.6" sp-trie = { version = "2.0.0", path = "../trie" } sp-core = { version = "2.0.0", path = "../core" } sp-panic-handler = { version = "2.0.0", path = "../panic-handler" } codec = { package = "parity-scale-codec", version = "1.0.0" } -sp-historical-data = { path = "../historical-data" } num-traits = "0.2.8" rand = "0.7.2" sp-externalities = { version = "2.0.0", path = "../externalities" } @@ -24,7 +24,6 @@ sp-externalities = { version = "2.0.0", path = "../externalities" } hex-literal = "0.2.1" criterion = "0.2" rand = { version = "0.7.1", features = ["small_rng"] } -sp-historical-data = { path = "../historical-data", features = [ "test-helpers" ] } [[bench]] name = "bench" @@ -32,6 +31,4 @@ harness = false [features] default = [] -test-helpers = [ - "sp-historical-data/test-helpers", -] +test-helpers = [] diff --git a/primitives/state-machine/fuzz/Cargo.lock b/primitives/state-machine/fuzz/Cargo.lock index 07fcc9c2b7617..3068dc88b3d8c 100644 --- a/primitives/state-machine/fuzz/Cargo.lock +++ b/primitives/state-machine/fuzz/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ [[package]] name = "ahash" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -245,14 +245,6 @@ dependencies = [ "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "elastic-array" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "environmental" version = "1.0.2" @@ -345,18 +337,10 @@ name = "hashbrown" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ahash 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)", + "ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "heapsize" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "hex" version = "0.4.0" @@ -491,12 +475,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "memory-db" -version = "0.15.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-util-mem 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-util-mem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -596,7 +581,7 @@ dependencies = [ [[package]] name = "parity-util-mem" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -969,6 +954,11 @@ dependencies = [ "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "smallvec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "sp-core" version = "2.0.0" @@ -1025,14 +1015,6 @@ dependencies = [ "sp-storage 2.0.0", ] -[[package]] -name = "sp-historical-data" -version = "2.0.0" -dependencies = [ - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-std 2.0.0", -] - [[package]] name = "sp-panic-handler" version = "2.0.0" @@ -1076,12 +1058,12 @@ dependencies = [ "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", "sp-core 2.0.0", "sp-externalities 2.0.0", - "sp-historical-data 2.0.0", "sp-panic-handler 2.0.0", "sp-trie 2.0.0", - "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1112,11 +1094,11 @@ name = "sp-trie" version = "2.0.0" dependencies = [ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "sp-core 2.0.0", "sp-std 2.0.0", - "trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1215,14 +1197,14 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.16.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1319,7 +1301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [metadata] "checksum Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -"checksum ahash 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "2f00e10d4814aa20900e7948174384f79f1317f24f0ba7494e735111653fc330" +"checksum ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" "checksum arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c7d1523aa3a127adf8b27af2404c03c12825b4c4d0698f01648d63fa9df62ee" "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" @@ -1351,7 +1333,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum curve25519-dalek 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum ed25519-dalek 1.0.0-pre.3 (registry+https://github.com/rust-lang/crates.io-index)" = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" -"checksum elastic-array 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "073be79b6538296faf81c631872676600616073817dd9a440c477ad09b408983" "checksum environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "34f8467a0284de039e6bd0e25c14519538462ba5beb548bb1f03e645097837a8" "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" @@ -1364,7 +1345,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" "checksum hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3bae29b6653b3412c2e71e9d486db9f9df5d701941d86683005efb9f2d28e3da" "checksum hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" -"checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" "checksum hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" "checksum hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" "checksum hmac-drbg 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" @@ -1382,7 +1362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum malloc_size_of_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e37c5d4cd9473c5f4c9c111f033f15d4df9bd378fdf615944e360a4f55a05f0b" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" -"checksum memory-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ef49315991403ba5fa225a70399df5e115f57b274cb0b1b4bcd6e734fa5bd783" +"checksum memory-db 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)" = "828bdf600636e90c56652689f7c3823ae2072104e4b0b5e83ea984f592f12ab9" "checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" "checksum merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "de2d16d3b15fec5943d1144f861f61f279d165fdd60998ca262913b9bf1c8adb" "checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" @@ -1394,7 +1374,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" "checksum parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f9f9d99dae413590a5f37e43cd99b94d4e62a244160562899126913ea7108673" "checksum parity-scale-codec-derive 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "492ac3aa93d6caa5d20e4e3e0b75d08e2dcd9dd8a50d19529548b6fe11b3f295" -"checksum parity-util-mem 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "570093f39f786beea92dcc09e45d8aae7841516ac19a50431953ac82a0e8f85c" +"checksum parity-util-mem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8174d85e62c4d615fddd1ef67966bdc5757528891d0742f15b131ad04667b3f9" "checksum parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" "checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" "checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" @@ -1437,6 +1417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "ca13fc1a832f793322228923fbb3aba9f3f44444898f835d31ad1b74fa0a2bf8" "checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +"checksum smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44e59e0c9fa00817912ae6e4e6e3c4fe04455e75699d06eedc7d85917ed8e8f4" "checksum static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" "checksum substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" "checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" @@ -1447,7 +1428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c5676413eaeb1ea35300a0224416f57abc3bd251657e0fafc12c47ff98c060" "checksum tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" "checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" -"checksum trie-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)" = "784a9813d23f18bccab728ab039c39b8a87d0d6956dcdece39e92f5cffe5076e" +"checksum trie-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)" = "191fda5d0106f3ed35a8c6875428b213e15c516e48129cc263dd7ad16e9a665f" "checksum trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0b779f7c1c8fe9276365d9d5be5c4b5adeacf545117bb3f64c974305789c5c0b" "checksum twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" "checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" diff --git a/primitives/state-machine/fuzz/src/lib.rs b/primitives/state-machine/fuzz/src/lib.rs index 4ec2774c2dea5..3c1a1bed0fe1e 100644 --- a/primitives/state-machine/fuzz/src/lib.rs +++ b/primitives/state-machine/fuzz/src/lib.rs @@ -36,6 +36,7 @@ fn fuzz_transactions_inner(input: &[u8], check_compactness: bool) { input_index += 1; (*v).into() } else { break }; + //println!("{:?}", action); actions.push(action); match action { @@ -240,10 +241,10 @@ impl RefOverlayedChanges { #[test] fn previous_fuzzed_error() { let inputs = [ - vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], - vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], vec![0x2,0xee,0xee,0x12,0x2,0x16,0x67,0xee,0xee,0xee,], vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], + vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], + vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], vec![50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67], vec![0x6e, 0xff, 0xf7, 0x0, 0x6e, 0xff, 0xff, 0x2d, 0xff, 0xff, 0xff, 0xe], diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index e9423638188d0..bcd74b0d21a9b 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -329,7 +329,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use sp_historical_data::sync_linear_transaction::{States, State, History, HistoricalEntry}; + use crate::{COMMITTED_LAYER, Layers, LayerEntry}; use super::*; const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); @@ -403,57 +403,57 @@ mod test { ]); let changes = OverlayedChanges { changes: OverlayedChangeSet { - states: States::test_state(0), + number_transactions: COMMITTED_LAYER + 1, top: vec![ - (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ + (EXTRINSIC_INDEX.to_vec(), Layers::from_iter(vec![ (OverlayedValue { value: Some(3u32.encode()), extrinsics: None, - }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), - (vec![100], History::from_iter(vec![ + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![100], Layers::from_iter(vec![ (OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) - }, State::Committed), + }, COMMITTED_LAYER), (OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) - }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), - (vec![101], History::from_iter(vec![ + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![101], Layers::from_iter(vec![ (OverlayedValue { value: Some(vec![203]), extrinsics: Some(vec![1].into_iter().collect()) - }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), - (vec![103], History::from_iter(vec![ + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![103], Layers::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![0, 1].into_iter().collect()) - }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), ].into_iter().collect(), children: vec![ (child_trie_key1, (vec![ - (vec![100], History::from_iter(vec![ + (vec![100], Layers::from_iter(vec![ (OverlayedValue { value: Some(vec![202]), extrinsics: Some(vec![3].into_iter().collect()) - }, State::Committed), + }, COMMITTED_LAYER), (OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![3, 0, 2].into_iter().collect()) - }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), ].into_iter().collect(), CHILD_INFO_1.to_owned())), (child_trie_key2, (vec![ - (vec![100], History::from_iter(vec![ + (vec![100], Layers::from_iter(vec![ (OverlayedValue { value: Some(vec![200]), extrinsics: Some(vec![0, 2].into_iter().collect()) - }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), ].into_iter().collect(), CHILD_INFO_2.to_owned())), ].into_iter().collect(), }, @@ -649,12 +649,12 @@ mod test { let (backend, storage, mut changes, config) = prepare_for_build(zero); // 110: missing from backend, set to None in overlay - changes.changes.top.insert(vec![110], History::from_iter(vec![ + changes.changes.top.insert(vec![110], Layers::from_iter(vec![ (OverlayedValue { value: None, extrinsics: Some(vec![1].into_iter().collect()), - }, State::Transaction(0)), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))); + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))); let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; let changes_trie_nodes = prepare_input( diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 640581bf4c14f..dffd293e6a642 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -659,35 +659,34 @@ mod tests { }, backend::InMemory, overlayed_changes::{OverlayedValue, OverlayedChangeSet}, }; use sp_core::storage::{Storage, StorageChild}; - use sp_historical_data::sync_linear_transaction::{History, HistoricalEntry, State}; + use crate::{Layers, LayerEntry, COMMITTED_LAYER}; type TestBackend = InMemory; type TestChangesTrieStorage = InMemoryChangesTrieStorage; type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend, TestChangesTrieStorage>; fn prepare_overlay_with_changes() -> OverlayedChanges { - OverlayedChanges { changes_trie_config: Some(ChangesTrieConfiguration { digest_interval: 0, digest_levels: 0, }), changes: OverlayedChangeSet { - states: Default::default(), + number_transactions: crate::COMMITTED_LAYER + 1, children: Default::default(), top: vec![ - (EXTRINSIC_INDEX.to_vec(), History::from_iter(vec![ + (EXTRINSIC_INDEX.to_vec(), Layers::from_iter(vec![ (OverlayedValue { value: Some(3u32.encode()), extrinsics: Some(vec![1].into_iter().collect()) - }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), - (vec![1], History::from_iter(vec![ + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![1], Layers::from_iter(vec![ (OverlayedValue { value: Some(vec![100]), extrinsics: Some(vec![1].into_iter().collect()) - }, State::Committed), - ].into_iter().map(|(value, index)| HistoricalEntry { value, index }))), + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), ].into_iter().collect(), }, } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 14bedd034e5d6..9591b08474ff2 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -728,6 +728,255 @@ fn try_read_overlay_value( } } +/// Stack of values in different transactional layers. +#[derive(Debug, Clone, PartialEq)] +struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); + +/// Index of commited layer, this is reserved. +const COMMITTED_LAYER: usize = 0; + +/// Start transaction stack layer at a size of two, +/// to only allocate at first transaction (we always +/// have a fix committed layer). +const ALLOCATED_HISTORY: usize = 2; + +impl Default for Layers { + fn default() -> Self { + Layers(Default::default()) + } +} + +/// An value entry of a indexed transactional layer. +#[derive(Debug, Clone, PartialEq)] +struct LayerEntry { + /// The stored value. + pub value: V, + /// The transactional layer index. + pub index: usize, +} + +impl From<(V, usize)> for LayerEntry { + fn from(input: (V, usize)) -> LayerEntry { + LayerEntry { value: input.0, index: input.1 } + } +} + +impl LayerEntry { + fn as_mut(&mut self) -> LayerEntry<&mut V> { + LayerEntry { value: &mut self.value, index: self.index } + } +} + +/// Possible results when updating `Layers` after a +/// transaction operation. +#[derive(Debug, PartialEq)] +enum LayeredOpsResult { + /// No inner data was changed, even technical + /// data, therefore no update is needed. + Unchanged, + /// Byte representation did change. + Changed, + /// No data is stored anymore, `Layers` can be dropped. + Cleared, +} + +impl Layers { + /// Push a value without checking without transactionallayer + /// consistency checks. + /// layer. + fn push_unchecked(&mut self, item: LayerEntry) { + self.0.push(item); + } + + /// Discard all prospective values, keeping previous committed value + /// only. + fn discard_prospective(&mut self) -> LayeredOpsResult { + if self.0.is_empty() { + return LayeredOpsResult::Cleared; + } + if self.0[0].index == COMMITTED_LAYER { + if self.0.len() == 1 { + LayeredOpsResult::Unchanged + } else { + self.0.truncate(1); + LayeredOpsResult::Changed + } + } else { + self.0.clear(); + LayeredOpsResult::Cleared + } + } + + /// Commits latest transaction pending value and clear existing transaction history. + pub fn commit_prospective(&mut self) -> LayeredOpsResult { + if self.0.is_empty() { + return LayeredOpsResult::Cleared; + } + if self.0.len() == 1 { + if self.0[0].index != COMMITTED_LAYER { + self.0[0].index = COMMITTED_LAYER; + } else { + return LayeredOpsResult::Unchanged; + } + } else if let Some(mut v) = self.0.pop() { + v.index = COMMITTED_LAYER; + self.0.clear(); + self.0.push(v); + } + LayeredOpsResult::Changed + } + + /// Access to the latest transactional value. + fn get(&self) -> Option<&V> { + self.0.last().map(|h| &h.value) + } + + /// Returns mutable handle on latest pending historical value. + fn get_mut(&mut self) -> Option> { + self.0.last_mut().map(|h| h.as_mut()) + } + + /// Set a new value, this function expect that + /// `number_transactions` is a valid state (can fail + /// otherwhise). + fn set(&mut self, number_transactions: usize, value: V) { + if let Some(v) = self.0.last_mut() { + debug_assert!(v.index <= number_transactions, + "Layers expects \ + only new values at the latest transaction \ + this indicates some unsynchronized layer value."); + if v.index == number_transactions { + v.value = value; + return; + } + } + self.0.push(LayerEntry { + value, + index: number_transactions, + }); + } + + /// Extracts the committed value if there is one. + pub fn into_committed(mut self) -> Option { + self.0.truncate(COMMITTED_LAYER + 1); + if let Some(LayerEntry { + value, + index: COMMITTED_LAYER, + }) = self.0.pop() { + return Some(value) + } else { + None + } + } + + /// Commit all transaction layer that are stacked + /// over the layer at `number_transaction`. + pub fn commit_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { + let mut new_value = None; + // Iterate on layers to get latest layered value and remove + // unused layered values inbetween. + for i in (0 .. self.0.len()).rev() { + if self.0[i].index > COMMITTED_LAYER { + if self.0[i].index > number_transaction { + // Remove value from committed layer + if let Some(v) = self.0.pop() { + if new_value.is_none() { + new_value = Some(v.value); + } + } + } else if self.0[i].index == number_transaction && new_value.is_some() { + // Remove parent layer value (will be overwritten by `new_value`. + self.0.pop(); + } else { + // Do not remove this is already a valid state. + break; + } + } else { + // Non transactional layer, stop removing history. + break; + } + } + if let Some(new_value) = new_value { + self.0.push(LayerEntry { + value: new_value, + index: number_transaction, + }); + return LayeredOpsResult::Changed; + } + LayeredOpsResult::Unchanged + } + + /// Discard value from transactional layers that are stacked over + /// the layer at `number_transaction`. + pub fn discard_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { + let init_len = self.0.len(); + let truncate_index = self.0.iter().rev() + .position(|entry| entry.index <= number_transaction) + .unwrap_or(init_len); + self.0.truncate(init_len - truncate_index); + if self.0.is_empty() { + LayeredOpsResult::Cleared + } else if self.0.len() != init_len { + LayeredOpsResult::Changed + } else { + LayeredOpsResult::Unchanged + } + } +} + +#[cfg(any(test, feature = "test-helpers"))] +impl Layers { + /// Get current number of stored historical values. + pub fn len(&self) -> usize { + self.0.len() + } +} + +#[cfg(test)] +impl Layers { + /// Create an history from a sequence of historical values. + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = Layers::default(); + for v in input { + history.push_unchecked(v); + } + history + } + + /// Get latest prospective value, excludes + /// committed values. + fn get_prospective(&self) -> Option<&V> { + match self.0.get(0) { + Some(entry) if entry.index == COMMITTED_LAYER => { + if let Some(entry) = self.0.get(1) { + if entry.index > COMMITTED_LAYER { + Some(&entry.value) + } else { + None + } + } else { + None + } + }, + Some(entry) => Some(&entry.value), + None => None, + } + } + + /// Get latest committed value. + fn get_committed(&self) -> Option<&V> { + if let Some(entry) = self.0.get(0) { + if entry.index == COMMITTED_LAYER { + return Some(&entry.value) + } else { + None + } + } else { + None + } + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 25acf93b25d40..8ed297f075c43 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -22,10 +22,7 @@ use std::collections::{HashMap, BTreeMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; -use sp_historical_data::sync_linear_transaction::{ - Layers, LayerEntry, -}; -use sp_historical_data::LayeredOpsResult; +use crate::{Layers, LayerEntry, LayeredOpsResult}; use std::ops; /// The overlayed changes to state to be queried on top of the backend. @@ -57,11 +54,11 @@ type TreeChangeSet = BTreeMap, Layers>; /// Overlayed change set, content keep trace of its history. /// /// Maps containing a linear history of each values are used. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { - /// Indexed state history. - pub(crate) number_transaction: usize, + /// Current number of stacked transaction. + pub(crate) number_transactions: usize, /// Top level storage changes. pub(crate) top: TreeChangeSet, /// Child storage changes. @@ -71,14 +68,23 @@ pub struct OverlayedChangeSet { pub(crate) children: HashMap, (TreeChangeSet, OwnedChildInfo)>, } +impl Default for OverlayedChangeSet { + fn default() -> Self { + OverlayedChangeSet { + number_transactions: crate::COMMITTED_LAYER + 1, + top: Default::default(), + children: Default::default(), + } + } +} + #[cfg(test)] impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { fn from_iter, OverlayedValue)>>(iter: T) -> Self { - use sp_historical_data::sync_linear_transaction::State; let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = Layers::default(); - history.push_unchecked(LayerEntry { value, index: State::Committed }); + history.push_unchecked(LayerEntry { value, index: crate::COMMITTED_LAYER }); history })).collect(); result @@ -110,9 +116,8 @@ fn set_with_extrinsic_inner_overlayed_value( value: Option>, extrinsic_index: u32, ) { - let state = number_transaction.as_state(); if let Some(current) = history.get_mut() { - if current.index == state { + if current.index == number_transaction { current.value.value = value; current.value.extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); @@ -121,7 +126,7 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); history.push_unchecked(LayerEntry { - index: state, + index: number_transaction, value: OverlayedValue { value, extrinsics, @@ -133,7 +138,7 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); history.push_unchecked(LayerEntry { - index: state, + index: number_transaction, value: OverlayedValue { value, extrinsics, @@ -150,50 +155,57 @@ impl OverlayedChangeSet { /// Discard prospective changes from the change set. pub fn discard_prospective(&mut self) { - self.number_transaction.discard_prospective(); - retain(&mut self.top, |_, history| States::apply_discard_prospective(history) != LayeredOpsResult::Cleared); + retain(&mut self.top, |_, history| history.discard_prospective() != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| States::apply_discard_prospective(history) != LayeredOpsResult::Cleared); + retain(map, |_, history| history.discard_prospective() != LayeredOpsResult::Cleared); !map.is_empty() }); + self.number_transactions = 1; } /// Commit prospective changes into the change set. pub fn commit_prospective(&mut self) { - self.number_transaction.commit_prospective(); - retain(&mut self.top, |_, history| States::apply_commit_prospective(history) != LayeredOpsResult::Cleared); + retain(&mut self.top, |_, history| history.commit_prospective() != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| States::apply_commit_prospective(history) != LayeredOpsResult::Cleared); + retain(map, |_, history| history.commit_prospective() != LayeredOpsResult::Cleared); !map.is_empty() }); + self.number_transactions = 1; } /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.number_transaction.start_transaction(); + self.number_transactions += 1; } /// Discard a transactional layer. /// There is always a transactional layer running /// (discarding the last trasactional layer open a new one). pub fn discard_transaction(&mut self) { - self.number_transaction.discard_transaction(); - let number_transaction = &self.number_transaction; - retain(&mut self.top, |_, history| number_transaction.apply_discard_transaction(history) != LayeredOpsResult::Cleared); + let number_transactions = self.number_transactions.saturating_sub(1); + + retain(&mut self.top, |_, history| history.discard_transaction(number_transactions) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| number_transaction.apply_discard_transaction(history) != LayeredOpsResult::Cleared); + retain(map, |_, history| history.discard_transaction(number_transactions) != LayeredOpsResult::Cleared); !map.is_empty() }); - self.number_transaction.finalize_discard(); + if number_transactions == crate::COMMITTED_LAYER { + self.number_transactions = crate::COMMITTED_LAYER + 1; + } else { + self.number_transactions = number_transactions; + } } /// Commit a transactional layer into previous transaction layer. pub fn commit_transaction(&mut self) { - self.number_transaction.commit_transaction(); - let number_transaction = &self.number_transaction; - retain(&mut self.top, |_, history| number_transaction.apply_commit_transaction(history) != LayeredOpsResult::Cleared); + if self.number_transactions > crate::COMMITTED_LAYER + 1 { + self.number_transactions -= 1; + } + + let number_transactions = self.number_transactions; + retain(&mut self.top, |_, history| history.commit_transaction(number_transactions) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| number_transaction.apply_commit_transaction(history) != LayeredOpsResult::Cleared); + retain(map, |_, history| history.commit_transaction(number_transactions) != LayeredOpsResult::Cleared); !map.is_empty() }); } @@ -372,7 +384,7 @@ impl OverlayedChanges { pub fn set_storage(&mut self, key: Vec, value: Option>) { let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - set_with_extrinsic_overlayed_value(entry, &self.changes.number_transaction, value, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.number_transactions, value, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -394,7 +406,7 @@ impl OverlayedChanges { let entry = map_entry.0.entry(key).or_default(); set_with_extrinsic_overlayed_value( entry, - &self.changes.number_transaction, + self.changes.number_transactions, value, extrinsic_index, ); @@ -412,7 +424,7 @@ impl OverlayedChanges { child_info: ChildInfo, ) { let extrinsic_index = self.extrinsic_index(); - let number_transaction = &self.changes.number_transaction; + let number_transaction = self.changes.number_transactions; let map_entry = self.changes.children.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); let updatable = map_entry.1.try_update(child_info); @@ -433,7 +445,7 @@ impl OverlayedChanges { for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - set_with_extrinsic_overlayed_value(entry, &self.changes.number_transaction, None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.number_transactions, None, extrinsic_index); } } } @@ -451,7 +463,7 @@ impl OverlayedChanges { for (key, entry) in child_change.0.iter_mut() { if key.starts_with(prefix) { - set_with_extrinsic_overlayed_value(entry, &self.changes.number_transaction, None, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, self.changes.number_transactions, None, extrinsic_index); } } } diff --git a/test-utils/runtime/Cargo.toml b/test-utils/runtime/Cargo.toml index eab428376c782..6444c7fcf1463 100644 --- a/test-utils/runtime/Cargo.toml +++ b/test-utils/runtime/Cargo.toml @@ -35,7 +35,6 @@ pallet-timestamp = { version = "2.0.0", default-features = false, path = "../../ sc-client = { version = "2.0.0", optional = true, path = "../../client" } sp-trie = { version = "2.0.0", default-features = false, path = "../../primitives/trie" } sp-transaction-pool = { version = "2.0.0", default-features = false, path = "../../primitives/transaction-pool" } -sp-historical-data = { path = "../../primitives/historical-data", default-features = false } trie-db = { version = "0.18.1", default-features = false } [dev-dependencies] @@ -80,6 +79,5 @@ std = [ "sc-client", "sp-trie/std", "sp-transaction-pool/std", - "sp-historical-data/std", "trie-db/std", ] diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index e2c5f8d161dbe..16138dbcc63a5 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -301,8 +301,6 @@ cfg_if! { fn fail_on_wasm() -> u64; /// trie no_std testing fn use_trie() -> u64; - /// History data no_std testing. - fn use_history_data(); /// Transactional tests. fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; @@ -558,10 +556,6 @@ cfg_if! { code_using_trie() } - fn use_history_data() { - test_historical_data() - } - fn use_transactions() -> u64 { system::test_transactions() } @@ -755,10 +749,6 @@ cfg_if! { code_using_trie() } - fn use_history_data() { - test_historical_data() - } - fn use_transactions() -> u64 { system::test_transactions() } @@ -1074,16 +1064,6 @@ mod tests { runtime_api.test_storage(&block_id).unwrap(); } - #[test] - fn use_history_data() { - let client = TestClientBuilder::new() - .set_execution_strategy(ExecutionStrategy::AlwaysWasm) - .build(); - let runtime_api = client.runtime_api(); - let block_id = BlockId::Number(client.chain_info().best_number); - assert!(runtime_api.use_history_data(&block_id).is_ok()); - } - #[test] fn test_transactions() { let client = TestClientBuilder::new() From 9f01802b0385251d69118283ef24c0c2ad614dfc Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 10 Jan 2020 15:01:27 +0100 Subject: [PATCH 112/129] Move transaction code in its own module. --- .../state-machine/src/changes_trie/build.rs | 2 +- .../src/changes_trie/build_cache.rs | 2 +- primitives/state-machine/src/ext.rs | 4 +- primitives/state-machine/src/lib.rs | 250 +---------------- .../state-machine/src/overlayed_changes.rs | 12 +- .../state-machine/src/transaction_layers.rs | 265 ++++++++++++++++++ 6 files changed, 276 insertions(+), 259 deletions(-) create mode 100644 primitives/state-machine/src/transaction_layers.rs diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index bcd74b0d21a9b..17c3bf4f91314 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -329,7 +329,7 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; - use crate::{COMMITTED_LAYER, Layers, LayerEntry}; + use crate::transaction_layers::{COMMITTED_LAYER, Layers, LayerEntry}; use super::*; const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); diff --git a/primitives/state-machine/src/changes_trie/build_cache.rs b/primitives/state-machine/src/changes_trie/build_cache.rs index 76e3652e16681..b7727f70d7d03 100644 --- a/primitives/state-machine/src/changes_trie/build_cache.rs +++ b/primitives/state-machine/src/changes_trie/build_cache.rs @@ -259,4 +259,4 @@ mod tests { assert_eq!(cache.changed_keys.len(), 0); } -} \ No newline at end of file +} diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index dffd293e6a642..5eaa7c6b1871f 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -659,7 +659,7 @@ mod tests { }, backend::InMemory, overlayed_changes::{OverlayedValue, OverlayedChangeSet}, }; use sp_core::storage::{Storage, StorageChild}; - use crate::{Layers, LayerEntry, COMMITTED_LAYER}; + use crate::transaction_layers::{Layers, LayerEntry, COMMITTED_LAYER}; type TestBackend = InMemory; type TestChangesTrieStorage = InMemoryChangesTrieStorage; @@ -672,7 +672,7 @@ mod tests { digest_levels: 0, }), changes: OverlayedChangeSet { - number_transactions: crate::COMMITTED_LAYER + 1, + number_transactions: COMMITTED_LAYER + 1, children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), Layers::from_iter(vec![ diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 9591b08474ff2..7ff499b3897a4 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -35,6 +35,7 @@ mod ext; mod testing; mod basic; mod overlayed_changes; +mod transaction_layers; mod proving_backend; mod trie_backend; mod trie_backend_essence; @@ -728,255 +729,6 @@ fn try_read_overlay_value( } } -/// Stack of values in different transactional layers. -#[derive(Debug, Clone, PartialEq)] -struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); - -/// Index of commited layer, this is reserved. -const COMMITTED_LAYER: usize = 0; - -/// Start transaction stack layer at a size of two, -/// to only allocate at first transaction (we always -/// have a fix committed layer). -const ALLOCATED_HISTORY: usize = 2; - -impl Default for Layers { - fn default() -> Self { - Layers(Default::default()) - } -} - -/// An value entry of a indexed transactional layer. -#[derive(Debug, Clone, PartialEq)] -struct LayerEntry { - /// The stored value. - pub value: V, - /// The transactional layer index. - pub index: usize, -} - -impl From<(V, usize)> for LayerEntry { - fn from(input: (V, usize)) -> LayerEntry { - LayerEntry { value: input.0, index: input.1 } - } -} - -impl LayerEntry { - fn as_mut(&mut self) -> LayerEntry<&mut V> { - LayerEntry { value: &mut self.value, index: self.index } - } -} - -/// Possible results when updating `Layers` after a -/// transaction operation. -#[derive(Debug, PartialEq)] -enum LayeredOpsResult { - /// No inner data was changed, even technical - /// data, therefore no update is needed. - Unchanged, - /// Byte representation did change. - Changed, - /// No data is stored anymore, `Layers` can be dropped. - Cleared, -} - -impl Layers { - /// Push a value without checking without transactionallayer - /// consistency checks. - /// layer. - fn push_unchecked(&mut self, item: LayerEntry) { - self.0.push(item); - } - - /// Discard all prospective values, keeping previous committed value - /// only. - fn discard_prospective(&mut self) -> LayeredOpsResult { - if self.0.is_empty() { - return LayeredOpsResult::Cleared; - } - if self.0[0].index == COMMITTED_LAYER { - if self.0.len() == 1 { - LayeredOpsResult::Unchanged - } else { - self.0.truncate(1); - LayeredOpsResult::Changed - } - } else { - self.0.clear(); - LayeredOpsResult::Cleared - } - } - - /// Commits latest transaction pending value and clear existing transaction history. - pub fn commit_prospective(&mut self) -> LayeredOpsResult { - if self.0.is_empty() { - return LayeredOpsResult::Cleared; - } - if self.0.len() == 1 { - if self.0[0].index != COMMITTED_LAYER { - self.0[0].index = COMMITTED_LAYER; - } else { - return LayeredOpsResult::Unchanged; - } - } else if let Some(mut v) = self.0.pop() { - v.index = COMMITTED_LAYER; - self.0.clear(); - self.0.push(v); - } - LayeredOpsResult::Changed - } - - /// Access to the latest transactional value. - fn get(&self) -> Option<&V> { - self.0.last().map(|h| &h.value) - } - - /// Returns mutable handle on latest pending historical value. - fn get_mut(&mut self) -> Option> { - self.0.last_mut().map(|h| h.as_mut()) - } - - /// Set a new value, this function expect that - /// `number_transactions` is a valid state (can fail - /// otherwhise). - fn set(&mut self, number_transactions: usize, value: V) { - if let Some(v) = self.0.last_mut() { - debug_assert!(v.index <= number_transactions, - "Layers expects \ - only new values at the latest transaction \ - this indicates some unsynchronized layer value."); - if v.index == number_transactions { - v.value = value; - return; - } - } - self.0.push(LayerEntry { - value, - index: number_transactions, - }); - } - - /// Extracts the committed value if there is one. - pub fn into_committed(mut self) -> Option { - self.0.truncate(COMMITTED_LAYER + 1); - if let Some(LayerEntry { - value, - index: COMMITTED_LAYER, - }) = self.0.pop() { - return Some(value) - } else { - None - } - } - - /// Commit all transaction layer that are stacked - /// over the layer at `number_transaction`. - pub fn commit_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { - let mut new_value = None; - // Iterate on layers to get latest layered value and remove - // unused layered values inbetween. - for i in (0 .. self.0.len()).rev() { - if self.0[i].index > COMMITTED_LAYER { - if self.0[i].index > number_transaction { - // Remove value from committed layer - if let Some(v) = self.0.pop() { - if new_value.is_none() { - new_value = Some(v.value); - } - } - } else if self.0[i].index == number_transaction && new_value.is_some() { - // Remove parent layer value (will be overwritten by `new_value`. - self.0.pop(); - } else { - // Do not remove this is already a valid state. - break; - } - } else { - // Non transactional layer, stop removing history. - break; - } - } - if let Some(new_value) = new_value { - self.0.push(LayerEntry { - value: new_value, - index: number_transaction, - }); - return LayeredOpsResult::Changed; - } - LayeredOpsResult::Unchanged - } - - /// Discard value from transactional layers that are stacked over - /// the layer at `number_transaction`. - pub fn discard_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { - let init_len = self.0.len(); - let truncate_index = self.0.iter().rev() - .position(|entry| entry.index <= number_transaction) - .unwrap_or(init_len); - self.0.truncate(init_len - truncate_index); - if self.0.is_empty() { - LayeredOpsResult::Cleared - } else if self.0.len() != init_len { - LayeredOpsResult::Changed - } else { - LayeredOpsResult::Unchanged - } - } -} - -#[cfg(any(test, feature = "test-helpers"))] -impl Layers { - /// Get current number of stored historical values. - pub fn len(&self) -> usize { - self.0.len() - } -} - -#[cfg(test)] -impl Layers { - /// Create an history from a sequence of historical values. - pub fn from_iter(input: impl IntoIterator>) -> Self { - let mut history = Layers::default(); - for v in input { - history.push_unchecked(v); - } - history - } - - /// Get latest prospective value, excludes - /// committed values. - fn get_prospective(&self) -> Option<&V> { - match self.0.get(0) { - Some(entry) if entry.index == COMMITTED_LAYER => { - if let Some(entry) = self.0.get(1) { - if entry.index > COMMITTED_LAYER { - Some(&entry.value) - } else { - None - } - } else { - None - } - }, - Some(entry) => Some(&entry.value), - None => None, - } - } - - /// Get latest committed value. - fn get_committed(&self) -> Option<&V> { - if let Some(entry) = self.0.get(0) { - if entry.index == COMMITTED_LAYER { - return Some(&entry.value) - } else { - None - } - } else { - None - } - } -} - #[cfg(test)] mod tests { use std::collections::BTreeMap; diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 8ed297f075c43..6903326168099 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -22,7 +22,7 @@ use std::collections::{HashMap, BTreeMap, BTreeSet}; use codec::Decode; use crate::changes_trie::{NO_EXTRINSIC_INDEX, Configuration as ChangesTrieConfig}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; -use crate::{Layers, LayerEntry, LayeredOpsResult}; +use crate::transaction_layers::{Layers, LayerEntry, LayeredOpsResult, COMMITTED_LAYER}; use std::ops; /// The overlayed changes to state to be queried on top of the backend. @@ -71,7 +71,7 @@ pub struct OverlayedChangeSet { impl Default for OverlayedChangeSet { fn default() -> Self { OverlayedChangeSet { - number_transactions: crate::COMMITTED_LAYER + 1, + number_transactions: COMMITTED_LAYER + 1, top: Default::default(), children: Default::default(), } @@ -84,7 +84,7 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = Layers::default(); - history.push_unchecked(LayerEntry { value, index: crate::COMMITTED_LAYER }); + history.push_unchecked(LayerEntry { value, index: COMMITTED_LAYER }); history })).collect(); result @@ -189,8 +189,8 @@ impl OverlayedChangeSet { retain(map, |_, history| history.discard_transaction(number_transactions) != LayeredOpsResult::Cleared); !map.is_empty() }); - if number_transactions == crate::COMMITTED_LAYER { - self.number_transactions = crate::COMMITTED_LAYER + 1; + if number_transactions == COMMITTED_LAYER { + self.number_transactions = COMMITTED_LAYER + 1; } else { self.number_transactions = number_transactions; } @@ -198,7 +198,7 @@ impl OverlayedChangeSet { /// Commit a transactional layer into previous transaction layer. pub fn commit_transaction(&mut self) { - if self.number_transactions > crate::COMMITTED_LAYER + 1 { + if self.number_transactions > COMMITTED_LAYER + 1 { self.number_transactions -= 1; } diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs new file mode 100644 index 0000000000000..69d4aec98ea37 --- /dev/null +++ b/primitives/state-machine/src/transaction_layers.rs @@ -0,0 +1,265 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Types and method for managing a stack of transactional values. + +/// Stack of values at different transactional layers. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); + +/// Index of reserved layer for commited values. +pub(crate) const COMMITTED_LAYER: usize = 0; + +/// Start transaction stack layer at a size of two, +/// to only allocate at first transaction (we always +/// have a fix committed layer). +const ALLOCATED_HISTORY: usize = 2; + +impl Default for Layers { + fn default() -> Self { + Layers(Default::default()) + } +} + +/// An value entry of a indexed transactional layer. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct LayerEntry { + /// The stored value. + pub value: V, + /// The transactional layer index. + pub index: usize, +} + +impl From<(V, usize)> for LayerEntry { + fn from(input: (V, usize)) -> LayerEntry { + LayerEntry { value: input.0, index: input.1 } + } +} + +impl LayerEntry { + fn as_mut(&mut self) -> LayerEntry<&mut V> { + LayerEntry { value: &mut self.value, index: self.index } + } +} + +/// Possible results when updating `Layers` after a +/// transaction operation. +#[derive(Debug, PartialEq)] +pub(crate) enum LayeredOpsResult { + /// No inner data was changed, even technical + /// data, therefore no update is needed. + Unchanged, + /// Byte representation did change. + Changed, + /// No data is stored anymore, `Layers` can be dropped. + Cleared, +} + +impl Layers { + /// Push a value without checking without transactional layer + /// consistency. + pub(crate) fn push_unchecked(&mut self, item: LayerEntry) { + self.0.push(item); + } + + /// Discard all prospective values, keeping previous committed value + /// only. + pub(crate) fn discard_prospective(&mut self) -> LayeredOpsResult { + if self.0.is_empty() { + return LayeredOpsResult::Cleared; + } + if self.0[0].index == COMMITTED_LAYER { + if self.0.len() == 1 { + LayeredOpsResult::Unchanged + } else { + self.0.truncate(1); + LayeredOpsResult::Changed + } + } else { + self.0.clear(); + LayeredOpsResult::Cleared + } + } + + /// Commits latest transaction pending value and clear existing transaction history. + pub fn commit_prospective(&mut self) -> LayeredOpsResult { + if self.0.is_empty() { + return LayeredOpsResult::Cleared; + } + if self.0.len() == 1 { + if self.0[0].index != COMMITTED_LAYER { + self.0[0].index = COMMITTED_LAYER; + } else { + return LayeredOpsResult::Unchanged; + } + } else if let Some(mut v) = self.0.pop() { + v.index = COMMITTED_LAYER; + self.0.clear(); + self.0.push(v); + } + LayeredOpsResult::Changed + } + + /// Access to the latest transactional value. + pub(crate) fn get(&self) -> Option<&V> { + self.0.last().map(|h| &h.value) + } + + /// Returns mutable handle on latest pending historical value. + pub(crate) fn get_mut(&mut self) -> Option> { + self.0.last_mut().map(|h| h.as_mut()) + } + + /// Set a new value, this function expect that + /// `number_transactions` is a valid state (can fail + /// otherwhise). + pub(crate) fn set(&mut self, number_transactions: usize, value: V) { + if let Some(v) = self.0.last_mut() { + debug_assert!(v.index <= number_transactions, + "Layers expects \ + only new values at the latest transaction \ + this indicates some unsynchronized layer value."); + if v.index == number_transactions { + v.value = value; + return; + } + } + self.0.push(LayerEntry { + value, + index: number_transactions, + }); + } + + /// Extracts the committed value if there is one. + pub fn into_committed(mut self) -> Option { + self.0.truncate(COMMITTED_LAYER + 1); + if let Some(LayerEntry { + value, + index: COMMITTED_LAYER, + }) = self.0.pop() { + return Some(value) + } else { + None + } + } + + /// Commit all transaction layer that are stacked + /// over the layer at `number_transaction`. + pub fn commit_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { + let mut new_value = None; + // Iterate on layers to get latest layered value and remove + // unused layered values inbetween. + for i in (0 .. self.0.len()).rev() { + if self.0[i].index > COMMITTED_LAYER { + if self.0[i].index > number_transaction { + // Remove value from committed layer + if let Some(v) = self.0.pop() { + if new_value.is_none() { + new_value = Some(v.value); + } + } + } else if self.0[i].index == number_transaction && new_value.is_some() { + // Remove parent layer value (will be overwritten by `new_value`. + self.0.pop(); + } else { + // Do not remove this is already a valid state. + break; + } + } else { + // Non transactional layer, stop removing history. + break; + } + } + if let Some(new_value) = new_value { + self.0.push(LayerEntry { + value: new_value, + index: number_transaction, + }); + return LayeredOpsResult::Changed; + } + LayeredOpsResult::Unchanged + } + + /// Discard value from transactional layers that are stacked over + /// the layer at `number_transaction`. + pub fn discard_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { + let init_len = self.0.len(); + let truncate_index = self.0.iter().rev() + .position(|entry| entry.index <= number_transaction) + .unwrap_or(init_len); + self.0.truncate(init_len - truncate_index); + if self.0.is_empty() { + LayeredOpsResult::Cleared + } else if self.0.len() != init_len { + LayeredOpsResult::Changed + } else { + LayeredOpsResult::Unchanged + } + } +} + +#[cfg(any(test, feature = "test-helpers"))] +impl Layers { + /// Get current number of stored historical values. + pub fn len(&self) -> usize { + self.0.len() + } +} + +#[cfg(test)] +impl Layers { + /// Create an history from a sequence of historical values. + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = Layers::default(); + for v in input { + history.push_unchecked(v); + } + history + } + + /// Get latest prospective value, excludes + /// committed values. + pub(crate) fn get_prospective(&self) -> Option<&V> { + match self.0.get(0) { + Some(entry) if entry.index == COMMITTED_LAYER => { + if let Some(entry) = self.0.get(1) { + if entry.index > COMMITTED_LAYER { + Some(&entry.value) + } else { + None + } + } else { + None + } + }, + Some(entry) => Some(&entry.value), + None => None, + } + } + + /// Get latest committed value. + pub(crate) fn get_committed(&self) -> Option<&V> { + if let Some(entry) = self.0.get(0) { + if entry.index == COMMITTED_LAYER { + return Some(&entry.value) + } else { + None + } + } else { + None + } + } +} From 92c99af845a914820860f8953d4296df99ecb7eb Mon Sep 17 00:00:00 2001 From: cheme Date: Fri, 10 Jan 2020 15:17:40 +0100 Subject: [PATCH 113/129] Move fuzzer base code to sp-state-machine to add non regression fuzzing test to ci. --- .../fuzz/fuzz_targets/fuzz_transactions.rs | 3 +- .../fuzz_transactions_with_compactness.rs | 3 +- primitives/state-machine/fuzz/src/lib.rs | 260 ------------------ primitives/state-machine/src/lib.rs | 3 +- .../state-machine/src/transaction_layers.rs | 257 ++++++++++++++++- 5 files changed, 255 insertions(+), 271 deletions(-) delete mode 100644 primitives/state-machine/fuzz/src/lib.rs diff --git a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs index 561fb38d4ced6..f105df16ad28b 100644 --- a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs +++ b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs @@ -1,7 +1,8 @@ #![no_main] use libfuzzer_sys::fuzz_target; +use sp_state_machine::transaction_layers_fuzz::fuzz_transactions_inner; fuzz_target!(|data: &[u8]| { - sp_state_machine_fuzz::fuzz_transactions(data) + fuzz_transactions_inner(data, false) }); diff --git a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs index 07f673627c968..585990a0a4bb2 100644 --- a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs +++ b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs @@ -1,7 +1,8 @@ #![no_main] use libfuzzer_sys::fuzz_target; +use sp_state_machine::transaction_layers_fuzz::fuzz_transactions_inner; fuzz_target!(|data: &[u8]| { - sp_state_machine_fuzz::fuzz_transactions_then_compactness(data) + fuzz_transactions_inner(data, true) }); diff --git a/primitives/state-machine/fuzz/src/lib.rs b/primitives/state-machine/fuzz/src/lib.rs deleted file mode 100644 index 3c1a1bed0fe1e..0000000000000 --- a/primitives/state-machine/fuzz/src/lib.rs +++ /dev/null @@ -1,260 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Substrate state machine fuzzing implementations. -use sp_state_machine::OverlayedChanges; -use std::collections::HashMap; - -/// Size of key, max 255 -const KEY_SPACE: u8 = 20; - -/// Size of key, max 255 -const VALUE_SPACE: u8 = 50; - -fn fuzz_transactions_inner(input: &[u8], check_compactness: bool) { - let mut input_index: usize = 0; - let mut overlayed = OverlayedChanges::default(); - let mut ref_overlayed = RefOverlayedChanges::default(); - - let mut actions = Vec::new(); - let mut values = Vec::new(); - loop { - let action: Actions = if let Some(v) = input.get(input_index) { - input_index += 1; - (*v).into() - } else { break }; - //println!("{:?}", action); - - actions.push(action); - match action { - Actions::CommitProspective => { - overlayed.commit_prospective(); - ref_overlayed.commit_prospective(); - }, - Actions::DropProspective => { - overlayed.discard_prospective(); - ref_overlayed.discard_prospective(); - }, - Actions::NewTransaction => { - overlayed.start_transaction(); - ref_overlayed.start_transaction(); - }, - Actions::CommitTransaction => { - overlayed.commit_transaction(); - ref_overlayed.commit_transaction(); - }, - Actions::DropTransaction => { - overlayed.discard_transaction(); - ref_overlayed.discard_transaction(); - }, - Actions::Insert => { - let key = if let Some(v) = input.get(input_index) { - input_index += 1; - v % KEY_SPACE - } else { break }; - let value = if let Some(v) = input.get(input_index) { - input_index += 1; - v % VALUE_SPACE - } else { break }; - values.push((key, value)); - overlayed.set_storage(vec![key], Some(vec![value])); - ref_overlayed.set_storage(vec![key], Some(vec![value])); - } - } - - } - - let mut success = true; - - let (check_value, len) = check_values(&overlayed, &ref_overlayed); - success &= check_value; - - if check_compactness { - let reference_size = ref_overlayed.total_length(); - let size = overlayed.top_count_keyvalue_pair(); - if reference_size != size { - println!("inconsistent gc {} {}", size, reference_size); - success = false; - } - let (check_value, len_compactness) = check_values(&overlayed, &ref_overlayed); - success &= check_value; - success &= len_compactness == len; - } - ref_overlayed.commit_prospective(); - let ref_len = ref_overlayed.committed.len(); - if len != ref_len { - println!("inconsistent length {} {}", len, ref_len); - success = false; - } - if !success { - println!("fuzzing: \n {:x?}", (&actions, &values)); - println!("input: \n {:?}", &input); - } - - assert!(success); -} - -fn check_values(overlayed: &OverlayedChanges, ref_overlayed: &RefOverlayedChanges) -> (bool, usize) { - let mut len = 0; - let mut success = true; - for (key, value) in overlayed.iter_values(None) { - let ref_value = ref_overlayed.storage(key); - if Some(value) != ref_value { - println!("at {:x?} different values {:x?} {:x?}", key, Some(value), ref_value); - success = false; - } - len += 1; - } - (success, len) -} - -pub fn fuzz_transactions(input: &[u8]) { - fuzz_transactions_inner(input, false); -} - -pub fn fuzz_transactions_then_compactness(input: &[u8]) { - fuzz_transactions_inner(input, true); -} - -#[derive(Clone, Copy, Debug)] -pub enum Actions { - Insert, - // Delete, same as an insert do not test. - CommitProspective, - DropProspective, - NewTransaction, - CommitTransaction, - DropTransaction, -} - -impl From for Actions { - fn from(v: u8) -> Self { - match (v as usize) * 100 / 255 { - v if v <= 5 => Actions::CommitProspective, - v if v <= 10 => Actions::DropProspective, - v if v <= 20 => Actions::NewTransaction, - v if v <= 30 => Actions::CommitTransaction, - v if v <= 40 => Actions::DropTransaction, - _ => Actions::Insert, - } - } -} - -/// A simple implementation of overlayed change -/// to use as a comparision. -/// It is partly incomplete (no child trie support, no change trie). -#[derive(Debug, Clone, Default)] -pub struct RefOverlayedChanges { - committed: HashMap, Vec>, - prospective: HashMap, Vec>, - transactions: Vec, Vec>>, -} - -impl RefOverlayedChanges { - pub fn discard_prospective(&mut self) { - self.transactions.clear(); - self.prospective.clear(); - } - - pub fn commit_prospective(&mut self) { - for _ in 0 .. self.transactions.len() { - self.commit_transaction(); - } - self.committed.extend(self.prospective.drain()); - } - - pub fn start_transaction(&mut self) { - self.transactions.push(Default::default()); - } - - pub fn discard_transaction(&mut self) { - if self.transactions.len() == 0 { - // clear prospective on no transaction started. - self.prospective.clear(); - } else { - let _ = self.transactions.pop(); - } - } - - /// Commit a transactional layer. - pub fn commit_transaction(&mut self) { - match self.transactions.len() { - 0 => (), - 1 => self.prospective.extend( - self.transactions.pop().expect("length just checked").into_iter() - ), - _ => { - let t = self.transactions.pop().expect("length just checked"); - self.transactions.last_mut().expect("length just checked") - .extend(t.into_iter()); - } - } - } - - pub fn set_storage(&mut self, key: Vec, val: Option>) { - if self.transactions.len() > 0 { - self.transactions.last_mut().expect("length just checked") - .insert(key, val.expect("fuzzer do not delete")); - } else { - self.prospective.insert(key, val.expect("fuzzer do not delete")); - } - } - - pub fn storage(&self, key: &[u8]) -> Option> { - for t in self.transactions.iter().rev() { - if let Some(v) = t.get(key) { - return Some(Some(v)); - } - } - if let Some(v) = self.prospective.get(key) { - return Some(Some(v)); - } - if let Some(v) = self.committed.get(key) { - return Some(Some(v)); - } - None - } - - pub fn total_length(&self) -> usize { - let tr_len: usize = self.transactions.iter() - .map(|l| l.len()).sum(); - self.committed.len() + self.prospective.len() + tr_len - } -} - -// Those are error previously found through fuzzing. -// They were fixed, but we keep the vectors to check -// for regression without needing to run fuzzing. -#[test] -fn previous_fuzzed_error() { - let inputs = [ - vec![0x2,0xee,0xee,0x12,0x2,0x16,0x67,0xee,0xee,0xee,], - vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], - vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], - vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], - vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], - vec![50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67], - vec![0x6e, 0xff, 0xf7, 0x0, 0x6e, 0xff, 0xff, 0x2d, 0xff, 0xff, 0xff, 0xe], - vec![0x2e,0x6e,0x22,0x32,0x2e,0x6e,0x22,0x32,0x3f,0x2e,], - vec![0xd9,0xff,0xfd,0x0,0xff,0xff,0xf8,0x1,0x92,0xff,0xbf,0x14,], - vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, - 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, - 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], - ]; - for input in inputs.iter() { - fuzz_transactions_inner(&input[..], true); - } -} diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 7ff499b3897a4..2250a6baa0acc 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -39,7 +39,8 @@ mod transaction_layers; mod proving_backend; mod trie_backend; mod trie_backend_essence; - +#[cfg(feature = "test-helpers")] +pub use transaction_layers::fuzz as transaction_layers_fuzz; pub use sp_trie::{trie_types::{Layout, TrieDBMut}, TrieMut, DBValue, MemoryDB}; pub use testing::TestExternalities; pub use basic::BasicExternalities; diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index 69d4aec98ea37..3f38b87af315b 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -211,14 +211,6 @@ impl Layers { } } -#[cfg(any(test, feature = "test-helpers"))] -impl Layers { - /// Get current number of stored historical values. - pub fn len(&self) -> usize { - self.0.len() - } -} - #[cfg(test)] impl Layers { /// Create an history from a sequence of historical values. @@ -263,3 +255,252 @@ impl Layers { } } } + +/// Base code for fuzzing. +#[cfg(any(test, feature = "test-helpers"))] +pub mod fuzz { + use crate::overlayed_changes::OverlayedChanges; + use std::collections::HashMap; + use super::Layers; + + /// Size of key, max 255 + const KEY_SPACE: u8 = 20; + + /// Size of key, max 255 + const VALUE_SPACE: u8 = 50; + + impl Layers { + /// Get current number of stored historical values. + pub fn len(&self) -> usize { + self.0.len() + } + } + + /// Fuzz input against a stack of hash map reference implementation. + /// `check_compactness` add a check in the number of stored entry. + pub fn fuzz_transactions_inner(input: &[u8], check_compactness: bool) { + let mut input_index: usize = 0; + let mut overlayed = OverlayedChanges::default(); + let mut ref_overlayed = RefOverlayedChanges::default(); + + let mut actions = Vec::new(); + let mut values = Vec::new(); + loop { + let action: Actions = if let Some(v) = input.get(input_index) { + input_index += 1; + (*v).into() + } else { break }; + //println!("{:?}", action); + + actions.push(action); + match action { + Actions::CommitProspective => { + overlayed.commit_prospective(); + ref_overlayed.commit_prospective(); + }, + Actions::DropProspective => { + overlayed.discard_prospective(); + ref_overlayed.discard_prospective(); + }, + Actions::NewTransaction => { + overlayed.start_transaction(); + ref_overlayed.start_transaction(); + }, + Actions::CommitTransaction => { + overlayed.commit_transaction(); + ref_overlayed.commit_transaction(); + }, + Actions::DropTransaction => { + overlayed.discard_transaction(); + ref_overlayed.discard_transaction(); + }, + Actions::Insert => { + let key = if let Some(v) = input.get(input_index) { + input_index += 1; + v % KEY_SPACE + } else { break }; + let value = if let Some(v) = input.get(input_index) { + input_index += 1; + v % VALUE_SPACE + } else { break }; + values.push((key, value)); + overlayed.set_storage(vec![key], Some(vec![value])); + ref_overlayed.set_storage(vec![key], Some(vec![value])); + } + } + + } + + let mut success = true; + + let (check_value, len) = check_values(&overlayed, &ref_overlayed); + success &= check_value; + + if check_compactness { + let reference_size = ref_overlayed.total_length(); + let size = overlayed.top_count_keyvalue_pair(); + if reference_size != size { + println!("inconsistent gc {} {}", size, reference_size); + success = false; + } + let (check_value, len_compactness) = check_values(&overlayed, &ref_overlayed); + success &= check_value; + success &= len_compactness == len; + } + ref_overlayed.commit_prospective(); + let ref_len = ref_overlayed.committed.len(); + if len != ref_len { + println!("inconsistent length {} {}", len, ref_len); + success = false; + } + if !success { + println!("fuzzing: \n {:x?}", (&actions, &values)); + println!("input: \n {:?}", &input); + } + + assert!(success); + } + + fn check_values(overlayed: &OverlayedChanges, ref_overlayed: &RefOverlayedChanges) -> (bool, usize) { + let mut len = 0; + let mut success = true; + for (key, value) in overlayed.iter_values(None) { + let ref_value = ref_overlayed.storage(key); + if Some(value) != ref_value { + println!("at {:x?} different values {:x?} {:x?}", key, Some(value), ref_value); + success = false; + } + len += 1; + } + (success, len) + } + + #[derive(Clone, Copy, Debug)] + enum Actions { + Insert, + // Delete, same as an insert do not test. + CommitProspective, + DropProspective, + NewTransaction, + CommitTransaction, + DropTransaction, + } + + impl From for Actions { + fn from(v: u8) -> Self { + match (v as usize) * 100 / 255 { + v if v <= 5 => Actions::CommitProspective, + v if v <= 10 => Actions::DropProspective, + v if v <= 20 => Actions::NewTransaction, + v if v <= 30 => Actions::CommitTransaction, + v if v <= 40 => Actions::DropTransaction, + _ => Actions::Insert, + } + } + } + + /// A simple implementation of overlayed change + /// to use as a comparision. + /// It is partly incomplete (no child trie support, no change trie). + #[derive(Debug, Clone, Default)] + struct RefOverlayedChanges { + committed: HashMap, Vec>, + prospective: HashMap, Vec>, + transactions: Vec, Vec>>, + } + + impl RefOverlayedChanges { + fn discard_prospective(&mut self) { + self.transactions.clear(); + self.prospective.clear(); + } + + fn commit_prospective(&mut self) { + for _ in 0 .. self.transactions.len() { + self.commit_transaction(); + } + self.committed.extend(self.prospective.drain()); + } + + fn start_transaction(&mut self) { + self.transactions.push(Default::default()); + } + + fn discard_transaction(&mut self) { + if self.transactions.len() == 0 { + // clear prospective on no transaction started. + self.prospective.clear(); + } else { + let _ = self.transactions.pop(); + } + } + + /// Commit a transactional layer. + fn commit_transaction(&mut self) { + match self.transactions.len() { + 0 => (), + 1 => self.prospective.extend( + self.transactions.pop().expect("length just checked").into_iter() + ), + _ => { + let t = self.transactions.pop().expect("length just checked"); + self.transactions.last_mut().expect("length just checked") + .extend(t.into_iter()); + } + } + } + + fn set_storage(&mut self, key: Vec, val: Option>) { + if self.transactions.len() > 0 { + self.transactions.last_mut().expect("length just checked") + .insert(key, val.expect("fuzzer do not delete")); + } else { + self.prospective.insert(key, val.expect("fuzzer do not delete")); + } + } + + fn storage(&self, key: &[u8]) -> Option> { + for t in self.transactions.iter().rev() { + if let Some(v) = t.get(key) { + return Some(Some(v)); + } + } + if let Some(v) = self.prospective.get(key) { + return Some(Some(v)); + } + if let Some(v) = self.committed.get(key) { + return Some(Some(v)); + } + None + } + + fn total_length(&self) -> usize { + let tr_len: usize = self.transactions.iter() + .map(|l| l.len()).sum(); + self.committed.len() + self.prospective.len() + tr_len + } + } + + // Those are samples which found error during fuzzing. + // They are kept as a low cust rust test for non regression purpose. + #[test] + fn previous_fuzzed_error() { + let inputs = [ + vec![0x2,0xee,0xee,0x12,0x2,0x16,0x67,0xee,0xee,0xee,], + vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], + vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], + vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], + vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], + vec![50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67], + vec![0x6e, 0xff, 0xf7, 0x0, 0x6e, 0xff, 0xff, 0x2d, 0xff, 0xff, 0xff, 0xe], + vec![0x2e,0x6e,0x22,0x32,0x2e,0x6e,0x22,0x32,0x3f,0x2e,], + vec![0xd9,0xff,0xfd,0x0,0xff,0xff,0xf8,0x1,0x92,0xff,0xbf,0x14,], + vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, + 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, + 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], + ]; + for input in inputs.iter() { + fuzz_transactions_inner(&input[..], true); + } + } +} From 26cfd0fbdb26e23df1e02dda43cf7a79d5601e06 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Mon, 13 Jan 2020 10:17:25 +0100 Subject: [PATCH 114/129] fix `where` formatting and some documentation from pr review. --- .../state-machine/src/overlayed_changes.rs | 19 ++++++++++++++----- .../state-machine/src/transaction_layers.rs | 6 +++--- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index a024710bd85d9..4ba0c627da07a 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -620,7 +620,10 @@ impl OverlayedChanges { changes_trie_storage: Option<&T>, parent_hash: H::Out, mut cache: StorageTransactionCache, - ) -> Result, String> where H::Out: Ord + Encode + 'static { + ) -> Result, String> + where + H::Out: Ord + Encode + 'static, + { // If the transaction does not exist, we generate it. if cache.transaction.is_none() { self.storage_root(backend, &mut cache); @@ -728,7 +731,8 @@ impl OverlayedChanges { backend: &B, cache: &mut StorageTransactionCache, ) -> H::Out - where H::Out: Ord + Encode, + where + H::Out: Ord + Encode, { let child_delta_iter = self.changes.owned_children_iter(); @@ -756,7 +760,10 @@ impl OverlayedChanges { parent_hash: H::Out, panic_on_storage_error: bool, cache: &mut StorageTransactionCache, - ) -> Result, ()> where H::Out: Ord + Encode + 'static { + ) -> Result, ()> + where + H::Out: Ord + Encode + 'static + { build_changes_trie::<_, T, H, N>( backend, changes_trie_storage, @@ -820,8 +827,10 @@ impl OverlayedChanges { /// hashmap usage. /// This could also be easilly replace if btreemap gets /// an implementation for retain in the future. -fn retain(map: &mut BTreeMap, mut f: F) where - F: FnMut(&K, &mut V) -> bool, { +fn retain(map: &mut BTreeMap, mut f: F) + where + F: FnMut(&K, &mut V) -> bool, +{ // this is use to discard some historical values when // their status is cleared. // Regarding this use case we will only remove individually. diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index 3f38b87af315b..b66e2e1937ebc 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -59,8 +59,8 @@ impl LayerEntry { /// transaction operation. #[derive(Debug, PartialEq)] pub(crate) enum LayeredOpsResult { - /// No inner data was changed, even technical - /// data, therefore no update is needed. + /// No inner value or metadata change occurs, + /// therefore no update is needed. Unchanged, /// Byte representation did change. Changed, @@ -125,7 +125,7 @@ impl Layers { /// Set a new value, this function expect that /// `number_transactions` is a valid state (can fail - /// otherwhise). + /// otherwise). pub(crate) fn set(&mut self, number_transactions: usize, value: V) { if let Some(v) = self.0.last_mut() { debug_assert!(v.index <= number_transactions, From ab272e3ec818fdcfee016c5ddca8578cd8e1c650 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 14 Jan 2020 12:22:14 +0100 Subject: [PATCH 115/129] Simplify logic for commit transaction. --- primitives/state-machine/src/overlayed_changes.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 4ba0c627da07a..c81856542cac0 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -277,15 +277,16 @@ impl OverlayedChangeSet { /// There is always a transactional layer running /// (discarding the last trasactional layer open a new one). pub fn discard_transaction(&mut self) { - let number_transactions = self.number_transactions.saturating_sub(1); - + self.number_transactions = self.number_transactions.saturating_sub(1); + let number_transactions = self.number_transactions; retain(&mut self.top, |_, history| history.discard_transaction(number_transactions) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { retain(map, |_, history| history.discard_transaction(number_transactions) != LayeredOpsResult::Cleared); !map.is_empty() }); if number_transactions == COMMITTED_LAYER { - self.number_transactions = COMMITTED_LAYER + 1; + // start new transaction + self.start_transaction(); } else { self.number_transactions = number_transactions; } @@ -293,10 +294,11 @@ impl OverlayedChangeSet { /// Commit a transactional layer into previous transaction layer. pub fn commit_transaction(&mut self) { - if self.number_transactions > COMMITTED_LAYER + 1 { - self.number_transactions -= 1; + if self.number_transactions == COMMITTED_LAYER + 1 { + // do nothing + return; } - + self.number_transactions = self.number_transactions.saturating_sub(1); let number_transactions = self.number_transactions; retain(&mut self.top, |_, history| history.commit_transaction(number_transactions) != LayeredOpsResult::Cleared); self.children.retain(|_, (map, _child_info)| { From 297009956e37ab85a554ccc5fc3fae5b05deb421 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 15 Jan 2020 19:26:00 +0100 Subject: [PATCH 116/129] provious lazy design for transaction. --- .../state-machine/src/changes_trie/build.rs | 9 +- primitives/state-machine/src/ext.rs | 4 +- primitives/state-machine/src/lib.rs | 9 +- .../state-machine/src/overlayed_changes.rs | 187 +++--- .../state-machine/src/transaction_layers.rs | 599 +++++++++++++----- 5 files changed, 574 insertions(+), 234 deletions(-) diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index c81fe12c7f714..00fea6c7c0910 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -330,6 +330,7 @@ mod test { use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; use crate::transaction_layers::{COMMITTED_LAYER, Layers, LayerEntry}; + use crate::transaction_layers::{States, TransactionState}; use super::*; const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); @@ -403,7 +404,11 @@ mod test { ]); let changes = OverlayedChanges { changes: OverlayedChangeSet { - number_transactions: COMMITTED_LAYER + 1, + states: States::test_vector( + vec![TransactionState::Pending, TransactionState::Pending], + vec![0, 0], + 1, + ), top: vec![ (EXTRINSIC_INDEX.to_vec(), Layers::from_iter(vec![ (OverlayedValue { @@ -458,6 +463,8 @@ mod test { ].into_iter().collect(), }, changes_trie_config: Some(config.clone()), + operation_from_last_gc: 0, + not_eager_gc: false, }; (backend, storage, changes, config) diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index e20db1fbe1706..a7b6014cfd88d 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -638,7 +638,7 @@ mod tests { digest_levels: 0, }), changes: OverlayedChangeSet { - number_transactions: COMMITTED_LAYER + 1, + states: Default::default(), children: Default::default(), top: vec![ (EXTRINSIC_INDEX.to_vec(), Layers::from_iter(vec![ @@ -655,6 +655,8 @@ mod tests { ].into_iter().map(|(value, index)| LayerEntry { value, index }))), ].into_iter().collect(), }, + operation_from_last_gc: 0, + not_eager_gc: false, } } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 8b5566e5b1dbd..d0bcd1c6b1908 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -317,11 +317,16 @@ impl<'a, B, H, N, T, Exec> StateMachine<'a, B, H, N, T, Exec> where CallResult, ) -> CallResult { - let orig_committed = self.overlay.changes.clone(); + let orig_committed = self.overlay.changes.states.committed(); + // disable eager gc to be able to rollback + self.overlay.not_eager_gc = true; let (result, was_native) = self.execute_aux(true, native_call.take()); + self.overlay.not_eager_gc = false; if was_native { - self.overlay.changes = orig_committed; + if result.is_ok() { + self.overlay.changes.states.unchecked_rollback_committed(orig_committed); + } let (wasm_result, _) = self.execute_aux( false, native_call, diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index c81856542cac0..00f53ff2e17f7 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -29,7 +29,8 @@ use std::iter::FromIterator; use std::collections::{HashMap, BTreeMap, BTreeSet}; use codec::{Decode, Encode}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; -use crate::transaction_layers::{Layers, LayerEntry, LayeredOpsResult, COMMITTED_LAYER}; +use crate::transaction_layers::{Layers, LayerEntry}; +use crate::transaction_layers::{States, DEFAULT_GC_CONF}; use std::ops; use hash_db::Hasher; @@ -45,6 +46,14 @@ pub struct OverlayedChanges { /// Changes trie configuration. None by default, but could be installed by the /// runtime if it supports change tries. pub(crate) changes_trie_config: Option, + /// Counter of number of operation between garbage collection. + /// Changing or deleting a value increment this counter by one. + /// An additional cost for data added is added for every n bytes of data. + /// n is currently defined by `DEFAULT_GC_CONF`. + pub(crate) operation_from_last_gc: usize, + /// Can be set to false to ensure we keep pre committed state (in case we want to move + /// back committed cursor). + pub(crate) not_eager_gc: bool, } /// The storage value, used inside OverlayedChanges. @@ -63,11 +72,11 @@ type TreeChangeSet = BTreeMap, Layers>; /// Overlayed change set, content keep trace of its history. /// /// Maps containing a linear history of each values are used. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { - /// Current number of stacked transaction. - pub(crate) number_transactions: usize, + /// Indexed state for transaction layers + pub(crate) states: States, /// Top level storage changes. pub(crate) top: TreeChangeSet, /// Child storage changes. @@ -77,16 +86,6 @@ pub struct OverlayedChangeSet { pub(crate) children: HashMap, (TreeChangeSet, OwnedChildInfo)>, } -impl Default for OverlayedChangeSet { - fn default() -> Self { - OverlayedChangeSet { - number_transactions: COMMITTED_LAYER + 1, - top: Default::default(), - children: Default::default(), - } - } -} - /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`]. /// /// This contains all the changes to the storage and transactions to apply theses changes to the @@ -179,7 +178,10 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { let mut result = OverlayedChangeSet::default(); result.top = iter.into_iter().map(|(k, value)| (k, { let mut history = Layers::default(); - history.push_unchecked(LayerEntry { value, index: COMMITTED_LAYER }); + history.push_unchecked(LayerEntry { + value, + index: crate::transaction_layers::COMMITTED_LAYER, + }); history })).collect(); result @@ -191,14 +193,14 @@ impl FromIterator<(Vec, OverlayedValue)> for OverlayedChangeSet { /// It does remove latest historical dropped items. fn set_with_extrinsic_overlayed_value( history: &mut Layers, - number_transaction: usize, + states: &States, value: Option>, extrinsic_index: Option, ) { if let Some(extrinsic) = extrinsic_index { - set_with_extrinsic_inner_overlayed_value(history, number_transaction, value, extrinsic) + set_with_extrinsic_inner_overlayed_value(history, states, value, extrinsic) } else { - history.set(number_transaction, OverlayedValue { + history.set(states, OverlayedValue { value, extrinsics: None, }) @@ -207,12 +209,13 @@ fn set_with_extrinsic_overlayed_value( fn set_with_extrinsic_inner_overlayed_value( history: &mut Layers, - number_transaction: usize, + states: &States, value: Option>, extrinsic_index: u32, ) { - if let Some(current) = history.get_mut() { - if current.index == number_transaction { + let state = states.num_states() - 1; + if let Some(current) = history.get_mut(states) { + if current.index == state { current.value.value = value; current.value.extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); @@ -221,7 +224,7 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); history.push_unchecked(LayerEntry { - index: number_transaction, + index: state, value: OverlayedValue { value, extrinsics, @@ -233,7 +236,7 @@ fn set_with_extrinsic_inner_overlayed_value( extrinsics.get_or_insert_with(Default::default) .insert(extrinsic_index); history.push_unchecked(LayerEntry { - index: number_transaction, + index: state, value: OverlayedValue { value, extrinsics, @@ -248,61 +251,40 @@ impl OverlayedChangeSet { self.top.is_empty() && self.children.is_empty() } - /// Discard prospective changes from the change set. + /// Discard prospective changes from the change set state. pub fn discard_prospective(&mut self) { - retain(&mut self.top, |_, history| history.discard_prospective() != LayeredOpsResult::Cleared); - self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| history.discard_prospective() != LayeredOpsResult::Cleared); - !map.is_empty() - }); - self.number_transactions = 1; + self.states.discard_prospective(); } - /// Commit prospective changes into the change set. + /// Commit prospective changes into the change set state. pub fn commit_prospective(&mut self) { - retain(&mut self.top, |_, history| history.commit_prospective() != LayeredOpsResult::Cleared); - self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| history.commit_prospective() != LayeredOpsResult::Cleared); - !map.is_empty() - }); - self.number_transactions = 1; + self.states.commit_prospective(); } /// Create a new transactional layer. pub fn start_transaction(&mut self) { - self.number_transactions += 1; + self.states.start_transaction(); } - /// Discard a transactional layer. + /// Discard a transactional layer state. /// There is always a transactional layer running /// (discarding the last trasactional layer open a new one). pub fn discard_transaction(&mut self) { - self.number_transactions = self.number_transactions.saturating_sub(1); - let number_transactions = self.number_transactions; - retain(&mut self.top, |_, history| history.discard_transaction(number_transactions) != LayeredOpsResult::Cleared); - self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| history.discard_transaction(number_transactions) != LayeredOpsResult::Cleared); - !map.is_empty() - }); - if number_transactions == COMMITTED_LAYER { - // start new transaction - self.start_transaction(); - } else { - self.number_transactions = number_transactions; - } + self.states.discard_transaction(); } /// Commit a transactional layer into previous transaction layer. pub fn commit_transaction(&mut self) { - if self.number_transactions == COMMITTED_LAYER + 1 { - // do nothing - return; - } - self.number_transactions = self.number_transactions.saturating_sub(1); - let number_transactions = self.number_transactions; - retain(&mut self.top, |_, history| history.commit_transaction(number_transactions) != LayeredOpsResult::Cleared); + self.states.commit_transaction(); + } + + /// Garbage collect. + fn gc(&mut self, prune_commit: bool) { + let states = &self.states; + // retain does change values + retain(&mut self.top, |_, history| history.get_mut_pruning(states, prune_commit).is_some()); self.children.retain(|_, (map, _child_info)| { - retain(map, |_, history| history.commit_transaction(number_transactions) != LayeredOpsResult::Cleared); + retain(map, |_, history| history.get_mut_pruning(states, prune_commit).is_some()); !map.is_empty() }); } @@ -325,7 +307,7 @@ impl OverlayedChangeSet { .into_iter() .flat_map(move |map| map.iter() .filter_map(move |(k, v)| - v.get().map(|v| (k.as_slice(), v))) + v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))) ), option_child_info) } @@ -351,7 +333,7 @@ impl OverlayedChangeSet { .map(move |(keyspace, child_content)| ( keyspace.as_slice(), child_content.0.iter() - .filter_map(move |(k, v)| v.get().map(|v| ( + .filter_map(move |(k, v)| v.get(self.states.as_ref()).map(|v| ( k.as_slice(), v.value.as_ref() .map(|v| v.as_slice()), @@ -374,7 +356,7 @@ impl OverlayedChangeSet { keyspace.to_vec(), map.iter() .filter_map(move |(k, v)| - v.get() + v.get(self.states.as_ref()) .map(|v| (k.to_vec(), v.value.as_ref().map(|v| v.to_vec())))), child_info.to_owned(), )) @@ -393,7 +375,7 @@ impl OverlayedChangeSet { keyspace.as_slice(), map.iter() .filter_map(move |(k, v)| - v.get().map(|v| (k.as_slice(), v))), + v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))), child_info.as_ref(), )) } @@ -405,7 +387,7 @@ impl OverlayedChangeSet { pub(crate) fn top_prospective(&self) -> BTreeMap, OverlayedValue> { let mut result = BTreeMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_prospective() { + if let Some(v) = v.get_prospective(&self.states) { result.insert(k.clone(), v.clone()); } } @@ -419,7 +401,7 @@ impl OverlayedChangeSet { pub(crate) fn top_committed(&self) -> BTreeMap, OverlayedValue> { let mut result = BTreeMap::new(); for (k, v) in self.top.iter() { - if let Some(v) = v.get_committed() { + if let Some(v) = v.get_committed(&self.states) { result.insert(k.clone(), v.clone()); } } @@ -454,7 +436,7 @@ impl OverlayedChanges { /// value has been set. pub fn storage(&self, key: &[u8]) -> Option> { if let Some(overlay_value) = self.changes.top.get(key) { - if let Some(o_value) = overlay_value.get() { + if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } @@ -467,7 +449,7 @@ impl OverlayedChanges { pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { if let Some(map) = self.changes.children.get(storage_key) { if let Some(overlay_value) = map.0.get(key) { - if let Some(o_value) = overlay_value.get() { + if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { return Some(o_value.value.as_ref().map(|v| v.as_slice())) } } @@ -479,9 +461,10 @@ impl OverlayedChanges { /// /// `None` can be used to delete a value specified by the given key. pub fn set_storage(&mut self, key: Vec, value: Option>) { + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); - set_with_extrinsic_overlayed_value(entry, self.changes.number_transactions, value, extrinsic_index); + set_with_extrinsic_overlayed_value(entry, &self.changes.states, value, extrinsic_index); } /// Inserts the given key-value pair into the prospective child change set. @@ -494,6 +477,7 @@ impl OverlayedChanges { key: Vec, value: Option>, ) { + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let map_entry = self.changes.children.entry(storage_key) .or_insert_with(|| (Default::default(), child_info.to_owned())); @@ -503,7 +487,7 @@ impl OverlayedChanges { let entry = map_entry.0.entry(key).or_default(); set_with_extrinsic_overlayed_value( entry, - self.changes.number_transactions, + &self.changes.states, value, extrinsic_index, ); @@ -521,14 +505,15 @@ impl OverlayedChanges { child_info: ChildInfo, ) { let extrinsic_index = self.extrinsic_index(); - let number_transaction = self.changes.number_transactions; + let states = &self.changes.states; let map_entry = self.changes.children.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); + self.operation_from_last_gc += map_entry.0.len(); let updatable = map_entry.1.try_update(child_info); debug_assert!(updatable); map_entry.0.values_mut() - .for_each(|e| set_with_extrinsic_overlayed_value(e, number_transaction, None, extrinsic_index)); + .for_each(|e| set_with_extrinsic_overlayed_value(e, states, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. @@ -540,11 +525,14 @@ impl OverlayedChanges { pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); + let mut number_removed = 0; for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - set_with_extrinsic_overlayed_value(entry, self.changes.number_transactions, None, extrinsic_index); + number_removed += 1; + set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } + self.operation_from_last_gc += number_removed; } pub(crate) fn clear_child_prefix( @@ -555,25 +543,36 @@ impl OverlayedChanges { ) { let extrinsic_index = self.extrinsic_index(); if let Some(child_change) = self.changes.children.get_mut(storage_key) { + let mut number_removed = 0; let updatable = child_change.1.try_update(child_info); debug_assert!(updatable); for (key, entry) in child_change.0.iter_mut() { if key.starts_with(prefix) { - set_with_extrinsic_overlayed_value(entry, self.changes.number_transactions, None, extrinsic_index); + number_removed += 1; + set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } + self.operation_from_last_gc += number_removed; } } /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.changes.discard_prospective(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { + self.operation_from_last_gc = 0; + self.gc(!self.not_eager_gc); + } } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { self.changes.commit_prospective(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { + self.operation_from_last_gc = 0; + self.gc(!self.not_eager_gc); + } } /// Create a new transactional layer. @@ -585,11 +584,19 @@ impl OverlayedChanges { /// A transaction is always running (history always end with pending). pub fn discard_transaction(&mut self) { self.changes.discard_transaction(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { + self.operation_from_last_gc = 0; + self.gc(false); + } } /// Commit a transactional layer. pub fn commit_transaction(&mut self) { self.changes.commit_transaction(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { + self.operation_from_last_gc = 0; + self.gc(false); + } } /// Consume `OverlayedChanges` and take committed set. @@ -599,17 +606,20 @@ impl OverlayedChanges { ){ let top = self.changes.top; let children = self.changes.children; + let committed = self.changes.states.committed(); + let states = self.changes.states.clone(); + let states2 = self.changes.states; ( top.into_iter() - .filter_map(move |(k, v)| v.into_committed() + .filter_map(move |(k, v)| v.into_committed(states.as_ref(), committed) .map(|v| (k, v.value))), - children.into_iter().map(move |(storage_key, child_content)| ( - storage_key, - child_content.0.into_iter() - .filter_map(move |(k, v)| v.into_committed() + children.into_iter().map(move |(storage_key, child_content)| { + let states2 = states2.clone(); + (storage_key, child_content.0.into_iter() + .filter_map(move |(k, v)| v.into_committed(states2.as_ref(), committed) .map(|v| (k, v.value))), - child_content.1.to_owned(), - )) + child_content.1.to_owned()) + }) ) } @@ -681,6 +691,8 @@ impl OverlayedChanges { let mut result = OverlayedChanges { changes, changes_trie_config, + operation_from_last_gc: 0, + not_eager_gc: false, }; committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); result.changes.commit_prospective(); @@ -714,7 +726,7 @@ impl OverlayedChanges { } #[cfg(any(test, feature = "test-helpers"))] - /// Count the number of key value pairs, at every history number_transaction. + /// Count the number of key value pairs, at every latest history. /// Should only be use for debugging or testing, this is slow /// and technical. pub fn top_count_keyvalue_pair(&self) -> usize { @@ -725,6 +737,13 @@ impl OverlayedChanges { result } + /// costy garbage collection of unneeded memory from + /// key values. Eager set to true will remove more + /// key value but allows more costy memory changes. + pub fn gc(&mut self, eager: bool) { + self.changes.gc(eager); + } + /// Generate the storage root using `backend` and all changes from `prospective` and `committed`. /// /// Returns the storage root and caches storage transaction in the given `cache`. @@ -796,7 +815,7 @@ impl OverlayedChanges { let mut next_keys = self.changes.top.range::<[u8], _>(range); while let Some((key, historic_value)) = next_keys.next() { - if let Some(overlay_value) = historic_value.get() { + if let Some(overlay_value) = historic_value.get(self.changes.states.as_ref()) { return Some((key, overlay_value)); } } @@ -815,7 +834,7 @@ impl OverlayedChanges { if let Some(child) = self.changes.children.get(storage_key) { let mut next_keys = child.0.range::<[u8], _>(range); while let Some((key, historic_value)) = next_keys.next() { - if let Some(overlay_value) = historic_value.get() { + if let Some(overlay_value) = historic_value.get(self.changes.states.as_ref()) { return Some((key, overlay_value)); } } diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index b66e2e1937ebc..9c0b55a488353 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -16,10 +16,177 @@ //! Types and method for managing a stack of transactional values. + +#[derive(Debug, Clone, Eq, PartialEq, Copy)] +/// State of a transactional layer. +pub(crate) enum TransactionState { + /// Data is under change and can still be dropped. + Pending, + /// Same as pending but does count as a transaction start. + TxPending, + /// Data pointing to this indexed historic state should + /// not be returned and can be removed. + Dropped, +} + +/// States for all past transaction layers. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct States { + history: Vec, + // Keep track of the transaction positions. + // This is redundant with `history`. + // This is only use to precalculate the result + // of a backward iteration to find the start + // of a transactional window (function + // `transaction_start_windows`). + // + // At many place we could use this instead + // of the `TxPending` state, but `TxPending` + // state is favored, except for `start_windows` + // case. + start_transaction_window: Vec, + // commit is the position in history of the latest committed. + commit: usize, +} + +impl Default for States { + fn default() -> Self { + States { + history: vec![TransactionState::Pending], + start_transaction_window: vec![0], + commit: 0, + } + } +} + +impl States { + /// Get reference of state, that is enough + /// information to query historical + /// data. + pub(crate) fn as_ref(&self) -> &[TransactionState] { + self.history.as_ref() + } + + /// Current number of inner states. + pub fn num_states(&self) -> usize { + self.history.len() + } + + /// Get index of committed layer. + pub fn committed(&self) -> usize { + self.commit + } + + /// Allow to rollback to a previous committed + /// index. + /// This can only work if there was no eager + /// garbage collection (eager collection can + /// change interval between `old_committed` and + /// `self.commit`). + pub fn unchecked_rollback_committed(&mut self, old_committed: usize) { + self.commit = old_committed; + self.discard_prospective(); + } + + /// Build any state for testing only. + #[cfg(test)] + pub(crate) fn test_vector( + history: Vec, + start_transaction_window: Vec, + commit: usize, + ) -> Self { + States { history, start_transaction_window, commit } + } + + /// Discard prospective changes to state. + /// That is revert all transaction up to the committed index. + pub fn discard_prospective(&mut self) { + for i in self.commit .. self.history.len() { + self.history[i] = TransactionState::Dropped; + self.start_transaction_window[i] = self.commit; + } + self.history.push(TransactionState::Pending); + self.start_transaction_window.push(self.commit); + } + + /// Commit prospective changes to state. + pub fn commit_prospective(&mut self) { + self.commit = self.history.len(); + self.history.push(TransactionState::Pending); + for i in 0..self.history.len() - 1 { + self.start_transaction_window[i] = self.commit; + } + self.start_transaction_window.push(self.commit); + } + + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.start_transaction_window.push(self.history.len()); + self.history.push(TransactionState::TxPending); + } + + /// Discard a transactional layer. + /// A transaction is always running (history always end with pending). + pub fn discard_transaction(&mut self) { + let mut i = self.history.len(); + while i > self.commit { + i -= 1; + match self.history[i] { + TransactionState::Dropped => (), + TransactionState::Pending => { + self.history[i] = TransactionState::Dropped; + }, + TransactionState::TxPending => { + self.history[i] = TransactionState::Dropped; + break; + }, + } + } + self.history.push(TransactionState::Pending); + self.start_transaction_window.truncate(i); + let previous = self.start_transaction_window.last() + .cloned().unwrap_or(self.commit); + self.start_transaction_window.resize(self.history.len(), previous); + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + let mut i = self.history.len(); + while i > self.commit { + i -= 1; + match self.history[i] { + TransactionState::Pending + | TransactionState::Dropped => (), + TransactionState::TxPending => { + self.history[i] = TransactionState::Pending; + break; + }, + } + } + self.history.push(TransactionState::Pending); + self.start_transaction_window.truncate(i); + let previous = self.start_transaction_window.last() + .cloned().unwrap_or(self.commit); + self.start_transaction_window.resize(self.history.len(), previous); + } +} + +#[inline] +/// Get previous index of pending state. +/// +/// Used to say if it is possible to drop a committed transaction +/// state value. +/// Committed index is seen as a transaction state. +pub fn transaction_start_windows(states: &States, from: usize) -> usize { + states.start_transaction_window[from] +} + /// Stack of values at different transactional layers. #[derive(Debug, Clone, PartialEq)] pub(crate) struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); +#[cfg(test)] /// Index of reserved layer for commited values. pub(crate) const COMMITTED_LAYER: usize = 0; @@ -50,164 +217,246 @@ impl From<(V, usize)> for LayerEntry { } impl LayerEntry { - fn as_mut(&mut self) -> LayerEntry<&mut V> { - LayerEntry { value: &mut self.value, index: self.index } + fn as_ref(&self) -> LayerEntry<&V> { + LayerEntry { value: &self.value, index: self.index } } } -/// Possible results when updating `Layers` after a -/// transaction operation. -#[derive(Debug, PartialEq)] -pub(crate) enum LayeredOpsResult { - /// No inner value or metadata change occurs, - /// therefore no update is needed. - Unchanged, - /// Byte representation did change. - Changed, - /// No data is stored anymore, `Layers` can be dropped. - Cleared, -} - impl Layers { + fn remove_until(&mut self, index: usize) { + if index > 0 { + if self.0.spilled() { + let owned = std::mem::replace(&mut self.0, Default::default()); + self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); + } else { + for i in (0..index).rev() { + self.0.remove(i); + } + } + } + } + + fn get_pending_unchecked( + &self, + states: &[TransactionState], + history_index: usize, + ) -> Option> { + let LayerEntry { value, index } = self.0[history_index].as_ref(); + match states[index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending => + return Some(LayerEntry { value, index }), + } + None + } + /// Push a value without checking without transactional layer /// consistency. pub(crate) fn push_unchecked(&mut self, item: LayerEntry) { self.0.push(item); } - /// Discard all prospective values, keeping previous committed value - /// only. - pub(crate) fn discard_prospective(&mut self) -> LayeredOpsResult { - if self.0.is_empty() { - return LayeredOpsResult::Cleared; + /// Access to the latest pending value (non dropped state). + /// When possible please prefer `get_mut` as it can free + /// some memory. + pub(crate) fn get(&self, states: &[TransactionState]) -> Option<&V> { + let self_len = self.0.len(); + if self_len == 0 { + return None; } - if self.0[0].index == COMMITTED_LAYER { - if self.0.len() == 1 { - LayeredOpsResult::Unchanged - } else { - self.0.truncate(1); - LayeredOpsResult::Changed + debug_assert!(states.len() >= self_len); + for index in (0 .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states, index) { + return Some(h.value); } - } else { - self.0.clear(); - LayeredOpsResult::Cleared } + None } - /// Commits latest transaction pending value and clear existing transaction history. - pub fn commit_prospective(&mut self) -> LayeredOpsResult { - if self.0.is_empty() { - return LayeredOpsResult::Cleared; + /// Access to latest pending value (non dropped state). + /// + /// This method removes latest dropped and merged values, + /// only keeping the latest valid value. + /// + /// Returns mutable latest pending value. + pub(crate) fn get_mut( + &mut self, + states: &States, + ) -> Option> { + let self_len = self.0.len(); + if self_len == 0 { + return None; } - if self.0.len() == 1 { - if self.0[0].index != COMMITTED_LAYER { - self.0[0].index = COMMITTED_LAYER; - } else { - return LayeredOpsResult::Unchanged; + debug_assert!(states.history.len() >= self_len); + let mut result = None; + let mut start_transaction_window = usize::max_value(); + let mut previous_switch = None; + for index in (0 .. self_len).rev() { + let state_index = self.0[index].index; + match states.history[state_index] { + TransactionState::TxPending => { + if state_index >= start_transaction_window { + previous_switch = Some(index); + } else { + if result.is_none() { + result = Some((index, state_index)); + } + } + break; + }, + TransactionState::Pending => { + if state_index >= start_transaction_window { + previous_switch = Some(index); + } else { + if result.is_none() { + result = Some((index, state_index)); + start_transaction_window = transaction_start_windows(states, state_index); + } else { + break; + } + } + }, + TransactionState::Dropped => (), } - } else if let Some(mut v) = self.0.pop() { - v.index = COMMITTED_LAYER; - self.0.clear(); - self.0.push(v); } - LayeredOpsResult::Changed + self.drop_last_values(result, previous_switch, 0) } - /// Access to the latest transactional value. - pub(crate) fn get(&self) -> Option<&V> { - self.0.last().map(|h| &h.value) + /// Method to prune regarding a full state. + /// It also returns the last value as mutable. + /// Internally it acts like `get_mut` with an + /// additional cleaning capacity to clean committed + /// state if `prune_to_commit` is set to true. + pub(crate) fn get_mut_pruning( + &mut self, + states: &States, + prune_to_commit: bool, + ) -> Option> { + let self_len = self.0.len(); + if self_len == 0 { + return None; + } + let mut prune_index = 0; + debug_assert!(states.history.len() >= self_len); + let mut result = None; + let mut start_transaction_window = usize::max_value(); + let mut previous_switch = None; + for index in (0 .. self_len).rev() { + let state_index = self.0[index].index; + match states.history[state_index] { + state @ TransactionState::TxPending + | state @ TransactionState::Pending => { + if state_index < states.commit && index > prune_index { + prune_index = index; + } + + if state_index >= start_transaction_window { + previous_switch = Some(index); + } else { + if result.is_none() { + result = Some((index, state_index)); + if state == TransactionState::Pending { + start_transaction_window = transaction_start_windows(states, state_index); + } + } else { + if prune_to_commit { + if state_index < states.commit { + break; + } + } else { + break; + } + } + } + }, + TransactionState::Dropped => (), + } + } + let deleted = if prune_to_commit && prune_index > 0 && result.is_some() { + self.remove_until(prune_index); + prune_index + } else { + 0 + }; + self.drop_last_values(result, previous_switch, deleted) } - /// Returns mutable handle on latest pending historical value. - pub(crate) fn get_mut(&mut self) -> Option> { - self.0.last_mut().map(|h| h.as_mut()) + // Function used to remove all last values for `get_mut` and + // `get_mut_pruning`. + // + // It expects the `result` of the iteration that lookup + // for the latest non dropped value (coupled with its state + // index). That is to remove terminal dropped states. + // + // It also does remove values on committed state that are + // not needed (before another committed value). + // `previous_switch` is the index to the first unneeded value. + // + // An index offset is here in case some content was `deleted` before + // this function call. + fn drop_last_values( + &mut self, + result: Option<(usize, usize)>, + previous_switch: Option, + deleted: usize, + ) -> Option> { + if let Some((index, state_index)) = result { + if index + 1 - deleted < self.0.len() { + self.0.truncate(index + 1 - deleted); + } + if let Some(switch_index) = previous_switch { + if let Some(mut value) = self.0.pop() { + self.0.truncate(switch_index - deleted); + value.index = state_index; + self.push_unchecked(value); + } + Some((&mut self.0[switch_index - deleted].value, state_index).into()) + } else { + Some((&mut self.0[index - deleted].value, state_index).into()) + } + } else { + self.0.clear(); + None + } } - /// Set a new value, this function expect that - /// `number_transactions` is a valid state (can fail - /// otherwise). - pub(crate) fn set(&mut self, number_transactions: usize, value: V) { - if let Some(v) = self.0.last_mut() { - debug_assert!(v.index <= number_transactions, - "Layers expects \ - only new values at the latest transaction \ - this indicates some unsynchronized layer value."); - if v.index == number_transactions { - v.value = value; + /// Set a value, it uses a state history as parameter. + /// + /// This method uses `get_mut` and does remove pending + /// dropped value. + pub(crate) fn set(&mut self, states: &States, value: V) { + let last_state_index = states.num_states() - 1; + if let Some(v) = self.get_mut(states) { + if v.index == last_state_index { + *v.value = value; return; } + debug_assert!(v.index < last_state_index, "Layers expects \ + only new values at the latest state"); } - self.0.push(LayerEntry { + self.push_unchecked(LayerEntry { value, - index: number_transactions, + index: last_state_index, }); } /// Extracts the committed value if there is one. - pub fn into_committed(mut self) -> Option { - self.0.truncate(COMMITTED_LAYER + 1); - if let Some(LayerEntry { - value, - index: COMMITTED_LAYER, - }) = self.0.pop() { - return Some(value) - } else { - None + pub fn into_committed(mut self, states: &[TransactionState], committed: usize) -> Option { + let self_len = self.0.len(); + if self_len == 0 { + return None; } - } - - /// Commit all transaction layer that are stacked - /// over the layer at `number_transaction`. - pub fn commit_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { - let mut new_value = None; - // Iterate on layers to get latest layered value and remove - // unused layered values inbetween. - for i in (0 .. self.0.len()).rev() { - if self.0[i].index > COMMITTED_LAYER { - if self.0[i].index > number_transaction { - // Remove value from committed layer - if let Some(v) = self.0.pop() { - if new_value.is_none() { - new_value = Some(v.value); - } - } - } else if self.0[i].index == number_transaction && new_value.is_some() { - // Remove parent layer value (will be overwritten by `new_value`. - self.0.pop(); - } else { - // Do not remove this is already a valid state. - break; + debug_assert!(states.len() >= self_len); + for index in (0 .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states, index) { + if h.index < committed { + self.0.truncate(index + 1); + return self.0.pop().map(|v| v.value); } - } else { - // Non transactional layer, stop removing history. - break; } } - if let Some(new_value) = new_value { - self.0.push(LayerEntry { - value: new_value, - index: number_transaction, - }); - return LayeredOpsResult::Changed; - } - LayeredOpsResult::Unchanged - } - - /// Discard value from transactional layers that are stacked over - /// the layer at `number_transaction`. - pub fn discard_transaction(&mut self, number_transaction: usize) -> LayeredOpsResult { - let init_len = self.0.len(); - let truncate_index = self.0.iter().rev() - .position(|entry| entry.index <= number_transaction) - .unwrap_or(init_len); - self.0.truncate(init_len - truncate_index); - if self.0.is_empty() { - LayeredOpsResult::Cleared - } else if self.0.len() != init_len { - LayeredOpsResult::Changed - } else { - LayeredOpsResult::Unchanged - } + None } } @@ -224,38 +473,96 @@ impl Layers { /// Get latest prospective value, excludes /// committed values. - pub(crate) fn get_prospective(&self) -> Option<&V> { - match self.0.get(0) { - Some(entry) if entry.index == COMMITTED_LAYER => { - if let Some(entry) = self.0.get(1) { - if entry.index > COMMITTED_LAYER { - Some(&entry.value) - } else { - None - } - } else { - None - } - }, - Some(entry) => Some(&entry.value), - None => None, + pub(crate) fn get_prospective(&self, states: &States) -> Option<&V> { + let self_len = self.len(); + if self_len == 0 { + return None; } + debug_assert!(states.history.len() >= self_len); + for index in (states.commit .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { + return Some(h.value); + } + } + None } /// Get latest committed value. - pub(crate) fn get_committed(&self) -> Option<&V> { - if let Some(entry) = self.0.get(0) { - if entry.index == COMMITTED_LAYER { - return Some(&entry.value) - } else { - None + pub(crate) fn get_committed(&self, states: &States) -> Option<&V> { + let self_len = self.len(); + if self_len == 0 { + return None; + } + debug_assert!(states.history.len() >= self_len); + for index in (0 .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { + if h.index < states.commit { + return Some(h.value); + } } - } else { - None } + None } } + +/// A default sample configuration to manage garbage collection +/// triggering. +pub(crate) const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { + trigger_transaction_gc: 1_000_000, + trigger_commit_gc: 100_000, + add_content_size_unit: 64, +}; + +/// Garbage collection configuration. +/// It is designed to listen on two different operation, transaction +/// and commit. +/// It match transcational semantic with transaction for transaction +/// operation and commit for prospective operation. +/// Considering transaction can be manipulated in extrinsic, this option +/// is quite unsafe and shoudl be use with care (the default value is +/// explicitelly very high). +/// More generally it should be very unlikelly that gc is needed +/// during a block processing for most use case. +/// +/// This is not a very good measurement and should be replace by +/// heap measurement or other metrics, at this point the gc even +/// if tested should not be required. +pub(crate) struct GCConfiguration { + /// Treshold in number of operation before running a garbage collection. + /// + /// Should be same as `TRIGGER_COMMIT_GC` or higher + /// (we most likely do not want lower as transaction are + /// possibly more frequent than commit). + pub trigger_transaction_gc: usize, + + /// Treshold in number of operation before running a garbage colletion + /// on a commit operation. + /// + /// We may want a lower value than for a transaction, even + /// a 1 if we want to do it between every operation. + pub trigger_commit_gc: usize, + + /// Used to count big content as multiple operations. + /// This is a number of octet. + /// Set to 0 to ignore. + pub add_content_size_unit: usize, +} + +impl GCConfiguration { + /// Cost depending on value if any. + pub fn operation_cost(&self, val: Option<&Vec>) -> usize { + let additional_cost = if self.add_content_size_unit > 0 { + if let Some(s) = val.as_ref() { + s.len() / self.add_content_size_unit + } else { + 0 + } + } else { 0 }; + 1 + additional_cost + } + +} /// Base code for fuzzing. #[cfg(any(test, feature = "test-helpers"))] pub mod fuzz { @@ -290,7 +597,6 @@ pub mod fuzz { input_index += 1; (*v).into() } else { break }; - //println!("{:?}", action); actions.push(action); match action { @@ -338,6 +644,7 @@ pub mod fuzz { if check_compactness { let reference_size = ref_overlayed.total_length(); + overlayed.gc(true); let size = overlayed.top_count_keyvalue_pair(); if reference_size != size { println!("inconsistent gc {} {}", size, reference_size); From ec7f6835ee79212d3834750009d025a1f07719ad Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 16 Jan 2020 09:47:26 +0100 Subject: [PATCH 117/129] Switch overlay_change primitives. --- .../state-machine/src/changes_trie/build.rs | 6 +- primitives/state-machine/src/lib.rs | 2 +- .../state-machine/src/overlayed_changes.rs | 73 +++++++++++++------ .../state-machine/src/transaction_layers.rs | 11 ++- 4 files changed, 62 insertions(+), 30 deletions(-) diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 00fea6c7c0910..022c6c91850c0 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -136,7 +136,7 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( // ignore temporary values (values that have null value at the end of operation // AND are not in storage at the beginning of operation if let Some(sk) = storage_key.as_ref() { - if !changes.child_storage(sk, k).map(|v| v.is_some()).unwrap_or_default() { + if !changes.child_storage_n(sk, k).map(|v| v.0.is_some()).unwrap_or_default() { if let Some(child_info) = child_info { if !backend.exists_child_storage(sk, child_info, k) .map_err(|e| format!("{}", e))? { @@ -145,7 +145,9 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( } } } else { - if !changes.storage(k).map(|v| v.is_some()).unwrap_or_default() { + // we use `storage_n` as this is called only a limited number of time + // per block processing. + if !changes.storage_n(k).map(|v| v.0.is_some()).unwrap_or_default() { if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { return Ok(map); } diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index d0bcd1c6b1908..30235d80e11a1 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -720,7 +720,7 @@ fn set_changes_trie_config( /// Reads storage value from overlay or from the backend. fn try_read_overlay_value( - overlay: &OverlayedChanges, + overlay: &mut OverlayedChanges, backend: &B, key: &[u8], ) -> Result>, Box> where H: Hasher, B: Backend { match overlay.storage(key).map(|x| x.map(|x| x.to_vec())) { diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 00f53ff2e17f7..bc3ed76f857e0 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -307,7 +307,7 @@ impl OverlayedChangeSet { .into_iter() .flat_map(move |map| map.iter() .filter_map(move |(k, v)| - v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))) + v.get(self.states.as_ref()).0.map(|v| (k.as_slice(), v))) ), option_child_info) } @@ -333,7 +333,7 @@ impl OverlayedChangeSet { .map(move |(keyspace, child_content)| ( keyspace.as_slice(), child_content.0.iter() - .filter_map(move |(k, v)| v.get(self.states.as_ref()).map(|v| ( + .filter_map(move |(k, v)| v.get(self.states.as_ref()).0.map(|v| ( k.as_slice(), v.value.as_ref() .map(|v| v.as_slice()), @@ -356,7 +356,7 @@ impl OverlayedChangeSet { keyspace.to_vec(), map.iter() .filter_map(move |(k, v)| - v.get(self.states.as_ref()) + v.get(self.states.as_ref()).0 .map(|v| (k.to_vec(), v.value.as_ref().map(|v| v.to_vec())))), child_info.to_owned(), )) @@ -375,7 +375,7 @@ impl OverlayedChangeSet { keyspace.as_slice(), map.iter() .filter_map(move |(k, v)| - v.get(self.states.as_ref()).map(|v| (k.as_slice(), v))), + v.get(self.states.as_ref()).0.map(|v| (k.as_slice(), v))), child_info.as_ref(), )) } @@ -431,13 +431,38 @@ impl OverlayedChanges { true } + /// Same as `storage` but to use carefully as it does not clean + /// tail values and complexity remain the same for following calls. + /// The return boolean indicates if a tail value iteration was required + pub fn storage_n(&self, key: &[u8]) -> Option<(Option<&[u8]>, bool)> { + if let Some(overlay_value) = self.changes.top.get(key) { + if let (Some(o_value), tail) = overlay_value.get(self.changes.states.as_ref()) { + return Some((o_value.value.as_ref().map(|v| v.as_slice()), tail)) + } + } + None + } + + /// Same as `child_storage` but to use carefully as it does not clean + /// tail values and complexity remain the same for following calls. + pub fn child_storage_n(&self, storage_key: &[u8], key: &[u8]) -> Option<(Option<&[u8]>, bool)> { + if let Some(map) = self.changes.children.get(storage_key) { + if let Some(overlay_value) = map.0.get(key) { + if let (Some(o_value), tail) = overlay_value.get(self.changes.states.as_ref()) { + return Some((o_value.value.as_ref().map(|v| v.as_slice()), tail)) + } + } + } + None + } + /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. - pub fn storage(&self, key: &[u8]) -> Option> { - if let Some(overlay_value) = self.changes.top.get(key) { - if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { - return Some(o_value.value.as_ref().map(|v| v.as_slice())) + pub fn storage(&mut self, key: &[u8]) -> Option> { + if let Some(overlay_value) = self.changes.top.get_mut(key) { + if let Some(o_value) = overlay_value.get_mut(&self.changes.states) { + return Some(o_value.value.value.as_ref().map(|v| v.as_slice())) } } None @@ -446,11 +471,11 @@ impl OverlayedChanges { /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. - pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { - if let Some(map) = self.changes.children.get(storage_key) { - if let Some(overlay_value) = map.0.get(key) { - if let Some(o_value) = overlay_value.get(self.changes.states.as_ref()) { - return Some(o_value.value.as_ref().map(|v| v.as_slice())) + pub fn child_storage(&mut self, storage_key: &[u8], key: &[u8]) -> Option> { + if let Some(map) = self.changes.children.get_mut(storage_key) { + if let Some(overlay_value) = map.0.get_mut(key) { + if let Some(o_value) = overlay_value.get_mut(&self.changes.states) { + return Some(o_value.value.value.as_ref().map(|v| v.as_slice())) } } } @@ -706,9 +731,11 @@ impl OverlayedChanges { /// set this index before first and unset after last extrinsic is executed. /// Changes that are made outside of extrinsics, are marked with /// `NO_EXTRINSIC_INDEX` index. - fn extrinsic_index(&self) -> Option { + fn extrinsic_index(&mut self) -> Option { match self.changes_trie_config.is_some() { true => Some( + // this call to storage is a map access for something that is just + // a counter and should be a field of the struct TODO self.storage(EXTRINSIC_INDEX) .and_then(|idx| idx.and_then(|idx| Decode::decode(&mut &*idx).ok())) .unwrap_or(NO_EXTRINSIC_INDEX)), @@ -810,13 +837,13 @@ impl OverlayedChanges { /// Returns the next (in lexicographic order) storage key in the overlayed alongside its value. /// If no value is next then `None` is returned. - pub fn next_storage_key_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { + pub fn next_storage_key_change(&mut self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded); - let mut next_keys = self.changes.top.range::<[u8], _>(range); + let mut next_keys = self.changes.top.range_mut::<[u8], _>(range); while let Some((key, historic_value)) = next_keys.next() { - if let Some(overlay_value) = historic_value.get(self.changes.states.as_ref()) { - return Some((key, overlay_value)); + if let Some(overlay_value) = historic_value.get_mut(&self.changes.states) { + return Some((key, overlay_value.value)); } } None @@ -825,17 +852,17 @@ impl OverlayedChanges { /// Returns the next (in lexicographic order) child storage key in the overlayed alongside its /// value. If no value is next then `None` is returned. pub fn next_child_storage_key_change( - &self, + &mut self, storage_key: &[u8], key: &[u8] ) -> Option<(&[u8], &OverlayedValue)> { let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded); - if let Some(child) = self.changes.children.get(storage_key) { - let mut next_keys = child.0.range::<[u8], _>(range); + if let Some(child) = self.changes.children.get_mut(storage_key) { + let mut next_keys = child.0.range_mut::<[u8], _>(range); while let Some((key, historic_value)) = next_keys.next() { - if let Some(overlay_value) = historic_value.get(self.changes.states.as_ref()) { - return Some((key, overlay_value)); + if let Some(overlay_value) = historic_value.get_mut(&self.changes.states) { + return Some((key, overlay_value.value)); } } } diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index 9c0b55a488353..35b70b3faba96 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -260,18 +260,21 @@ impl Layers { /// Access to the latest pending value (non dropped state). /// When possible please prefer `get_mut` as it can free /// some memory. - pub(crate) fn get(&self, states: &[TransactionState]) -> Option<&V> { + /// If tail freeing is needed, the function also return true + /// (this indicates the operation run suboptimally and could + /// in certain condition leads to bad complexity). + pub(crate) fn get(&self, states: &[TransactionState]) -> (Option<&V>, bool) { let self_len = self.0.len(); if self_len == 0 { - return None; + return (None, false); } debug_assert!(states.len() >= self_len); for index in (0 .. self_len).rev() { if let Some(h) = self.get_pending_unchecked(states, index) { - return Some(h.value); + return (Some(h.value), index != self_len - 1); } } - None + (None, true) } /// Access to latest pending value (non dropped state). From 68b96789443691faa91f5a8dd7e41fba46d36c62 Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 16 Jan 2020 10:30:29 +0100 Subject: [PATCH 118/129] Switch to mutable for storage, child storage and similar. --- primitives/externalities/src/lib.rs | 18 ++--- primitives/io/src/lib.rs | 12 ++-- primitives/state-machine/src/basic.rs | 22 +++---- primitives/state-machine/src/ext.rs | 94 ++++++++++++++------------- 4 files changed, 76 insertions(+), 70 deletions(-) diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 133ca708eb241..6cd7dc898893e 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -37,16 +37,16 @@ mod scope_limited; /// Provides access to the storage and to other registered extensions. pub trait Externalities: ExtensionStore { /// Read runtime storage. - fn storage(&self, key: &[u8]) -> Option>; + fn storage(&mut self, key: &[u8]) -> Option>; /// Get storage value hash. This may be optimized for large values. - fn storage_hash(&self, key: &[u8]) -> Option>; + fn storage_hash(&mut self, key: &[u8]) -> Option>; /// Get child storage value hash. This may be optimized for large values. /// /// Returns an `Option` that holds the SCALE encoded hash. fn child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -76,7 +76,7 @@ pub trait Externalities: ExtensionStore { /// /// Returns an `Option` that holds the SCALE encoded hash. fn original_child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -86,7 +86,7 @@ pub trait Externalities: ExtensionStore { /// /// Returns an `Option` that holds the SCALE encoded hash. fn child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -124,13 +124,13 @@ pub trait Externalities: ExtensionStore { } /// Whether a storage entry exists. - fn exists_storage(&self, key: &[u8]) -> bool { + fn exists_storage(&mut self, key: &[u8]) -> bool { self.storage(key).is_some() } /// Whether a child storage entry exists. fn exists_child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -139,11 +139,11 @@ pub trait Externalities: ExtensionStore { } /// Returns the key immediately following the given key, if it exists. - fn next_storage_key(&self, key: &[u8]) -> Option>; + fn next_storage_key(&mut self, key: &[u8]) -> Option>; /// Returns the key immediately following the given key, if it exists, in child storage. fn next_child_storage_key( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 0ffe4080de418..7db340ebfd63e 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -86,7 +86,7 @@ fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { #[runtime_interface] pub trait Storage { /// Returns the data for `key` in the storage or `None` if the key can not be found. - fn get(&self, key: &[u8]) -> Option> { + fn get(&mut self, key: &[u8]) -> Option> { self.storage(key).map(|s| s.to_vec()) } @@ -102,7 +102,7 @@ pub trait Storage { /// This function specifically returns the data for `key` in the child storage or `None` /// if the key can not be found. fn child_get( - &self, + &mut self, child_storage_key: &[u8], child_definition: &[u8], child_type: u32, @@ -119,7 +119,7 @@ pub trait Storage { /// doesn't exist at all. /// If `value_out` length is smaller than the returned length, only `value_out` length bytes /// are copied into `value_out`. - fn read(&self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { + fn read(&mut self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { self.storage(key).map(|value| { let value_offset = value_offset as usize; let data = &value[value_offset.min(value.len())..]; @@ -137,7 +137,7 @@ pub trait Storage { /// /// See `child_get` for common child api parameters. fn child_read( - &self, + &mut self, child_storage_key: &[u8], child_definition: &[u8], child_type: u32, @@ -217,7 +217,7 @@ pub trait Storage { } /// Check whether the given `key` exists in storage. - fn exists(&self, key: &[u8]) -> bool { + fn exists(&mut self, key: &[u8]) -> bool { self.exists_storage(key) } @@ -225,7 +225,7 @@ pub trait Storage { /// /// See `child_get` for common child api parameters. fn child_exists( - &self, + &mut self, child_storage_key: &[u8], child_definition: &[u8], child_type: u32, diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index e0a73dd8001a2..0ac3124ac90c1 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -111,24 +111,24 @@ impl From, Vec>> for BasicExternalities { } impl Externalities for BasicExternalities { - fn storage(&self, key: &[u8]) -> Option> { + fn storage(&mut self, key: &[u8]) -> Option> { self.inner.top.get(key).cloned() } - fn storage_hash(&self, key: &[u8]) -> Option> { + fn storage_hash(&mut self, key: &[u8]) -> Option> { self.storage(key).map(|v| Blake2Hasher::hash(&v).encode()) } fn original_storage(&self, key: &[u8]) -> Option> { - self.storage(key) + self.inner.top.get(key).cloned() } fn original_storage_hash(&self, key: &[u8]) -> Option> { - self.storage_hash(key) + self.original_storage(key).map(|v| Blake2Hasher::hash(&v).encode()) } fn child_storage( - &self, + &mut self, storage_key: ChildStorageKey, _child_info: ChildInfo, key: &[u8], @@ -137,7 +137,7 @@ impl Externalities for BasicExternalities { } fn child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -146,7 +146,7 @@ impl Externalities for BasicExternalities { } fn original_child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -157,19 +157,19 @@ impl Externalities for BasicExternalities { fn original_child_storage( &self, storage_key: ChildStorageKey, - child_info: ChildInfo, + _child_info: ChildInfo, key: &[u8], ) -> Option> { - Externalities::child_storage(self, storage_key, child_info, key) + self.inner.children.get(storage_key.as_ref()).and_then(|child| child.data.get(key)).cloned() } - fn next_storage_key(&self, key: &[u8]) -> Option> { + fn next_storage_key(&mut self, key: &[u8]) -> Option> { let range = (Bound::Excluded(key), Bound::Unbounded); self.inner.top.range::<[u8], _>(range).next().map(|(k, _)| k).cloned() } fn next_child_storage_key( - &self, + &mut self, storage_key: ChildStorageKey, _child_info: ChildInfo, key: &[u8], diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index a7b6014cfd88d..bc653fd1a98c7 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -153,7 +153,7 @@ where T: 'a + ChangesTrieStorage, N: crate::changes_trie::BlockNumber, { - fn storage(&self, key: &[u8]) -> Option> { + fn storage(&mut self, key: &[u8]) -> Option> { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); @@ -165,7 +165,7 @@ where result } - fn storage_hash(&self, key: &[u8]) -> Option> { + fn storage_hash(&mut self, key: &[u8]) -> Option> { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay .storage(key) @@ -205,7 +205,7 @@ where } fn child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -230,7 +230,7 @@ where } fn child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, _child_info: ChildInfo, key: &[u8], @@ -275,7 +275,7 @@ where } fn original_child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -294,7 +294,7 @@ where result.map(|r| r.encode()) } - fn exists_storage(&self, key: &[u8]) -> bool { + fn exists_storage(&mut self, key: &[u8]) -> bool { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = match self.overlay.storage(key) { Some(x) => x.is_some(), @@ -311,7 +311,7 @@ where } fn exists_child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -334,48 +334,54 @@ where result } - fn next_storage_key(&self, key: &[u8]) -> Option> { - let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); - let next_overlay_key_change = self.overlay.next_storage_key_change(key); + fn next_storage_key(&mut self, key: &[u8]) -> Option> { + let next_key = { + let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); + let next_overlay_key_change = self.overlay.next_storage_key_change(key); - match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), - (backend_key, None) => backend_key, - (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { - Some(overlay_key.0.to_vec()) - } else { - self.next_storage_key(&overlay_key.0[..]) - }, - } + match (next_backend_key, next_overlay_key_change) { + (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => return Some(backend_key), + (backend_key, None) => return backend_key, + (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { + return Some(overlay_key.0.to_vec()) + } else { + // TODO make this function non recursive to avoid this clone + overlay_key.0.to_vec() + }, + } + }; + self.next_storage_key(&next_key[..]) } fn next_child_storage_key( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], ) -> Option> { - let next_backend_key = self.backend - .next_child_storage_key(storage_key.as_ref(), child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL); - let next_overlay_key_change = self.overlay.next_child_storage_key_change( - storage_key.as_ref(), - key - ); + let next_key = { + let next_backend_key = self.backend + .next_child_storage_key(storage_key.as_ref(), child_info, key) + .expect(EXT_NOT_ALLOWED_TO_FAIL); + let next_overlay_key_change = self.overlay.next_child_storage_key_change( + storage_key.as_ref(), + key + ); - match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), - (backend_key, None) => backend_key, - (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { - Some(overlay_key.0.to_vec()) - } else { - self.next_child_storage_key( - storage_key, - child_info, - &overlay_key.0[..], - ) - }, - } + match (next_backend_key, next_overlay_key_change) { + (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => return Some(backend_key), + (backend_key, None) => return backend_key, + (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { + return Some(overlay_key.0.to_vec()) + } else { + overlay_key.0.to_vec() + }, + } + }; + // TODO no need to query child at each recursive iter here + // and also find a way to remove the clone (non recursive and global mut handle should do the + // trick). + self.next_child_storage_key(storage_key, child_info, &next_key[..]) } fn place_storage(&mut self, key: Vec, value: Option>) { @@ -722,7 +728,7 @@ mod tests { children: map![] }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_storage_key(&[5]), Some(vec![10])); @@ -738,7 +744,7 @@ mod tests { drop(ext); overlay.set_storage(vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_storage_key(&[40]), Some(vec![50])); @@ -771,7 +777,7 @@ mod tests { }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[5]), Some(vec![10])); @@ -787,7 +793,7 @@ mod tests { drop(ext); overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[40]), Some(vec![50])); From 47f117abf2458ccf1effc18dbd944fab671499e7 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 20 Jan 2020 17:59:21 +0100 Subject: [PATCH 119/129] Fix some doc, mark loop enhancement to test. --- primitives/state-machine/fuzz/Cargo.lock | 44 ++++---- .../state-machine/src/changes_trie/build.rs | 6 +- .../state-machine/src/changes_trie/mod.rs | 4 +- .../state-machine/src/overlayed_changes.rs | 36 ++++--- .../state-machine/src/transaction_layers.rs | 102 +++++++++++------- 5 files changed, 113 insertions(+), 79 deletions(-) diff --git a/primitives/state-machine/fuzz/Cargo.lock b/primitives/state-machine/fuzz/Cargo.lock index 3068dc88b3d8c..fd8a67dc7947d 100644 --- a/primitives/state-machine/fuzz/Cargo.lock +++ b/primitives/state-machine/fuzz/Cargo.lock @@ -247,7 +247,7 @@ dependencies = [ [[package]] name = "environmental" -version = "1.0.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -283,7 +283,7 @@ dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -475,7 +475,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "memory-db" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", @@ -867,7 +867,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rustc-hex" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -980,12 +980,12 @@ dependencies = [ "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "sp-debug-derive 2.0.0", - "sp-externalities 2.0.0", + "sp-externalities 0.8.0", "sp-runtime-interface 2.0.0", "sp-std 2.0.0", "sp-storage 2.0.0", @@ -1008,9 +1008,9 @@ dependencies = [ [[package]] name = "sp-externalities" -version = "2.0.0" +version = "0.8.0" dependencies = [ - "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "environmental 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "sp-std 2.0.0", "sp-storage 2.0.0", ] @@ -1027,10 +1027,9 @@ dependencies = [ name = "sp-runtime-interface" version = "2.0.0" dependencies = [ - "environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-externalities 2.0.0", + "sp-externalities 0.8.0", "sp-runtime-interface-proc-macro 2.0.0", "sp-std 2.0.0", "sp-wasm-interface 2.0.0", @@ -1050,7 +1049,7 @@ dependencies = [ [[package]] name = "sp-state-machine" -version = "2.0.0" +version = "0.8.0" dependencies = [ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1060,10 +1059,10 @@ dependencies = [ "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", "sp-core 2.0.0", - "sp-externalities 2.0.0", + "sp-externalities 0.8.0", "sp-panic-handler 2.0.0", "sp-trie 2.0.0", - "trie-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1072,7 +1071,7 @@ name = "sp-state-machine-fuzz" version = "0.0.1" dependencies = [ "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", - "sp-state-machine 2.0.0", + "sp-state-machine 0.8.0", ] [[package]] @@ -1094,11 +1093,11 @@ name = "sp-trie" version = "2.0.0" dependencies = [ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memory-db 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memory-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "sp-core 2.0.0", "sp-std 2.0.0", - "trie-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1197,13 +1196,14 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.18.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1235,7 +1235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1333,7 +1333,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum curve25519-dalek 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum ed25519-dalek 1.0.0-pre.3 (registry+https://github.com/rust-lang/crates.io-index)" = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" -"checksum environmental 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "34f8467a0284de039e6bd0e25c14519538462ba5beb548bb1f03e645097837a8" +"checksum environmental 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" @@ -1362,7 +1362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum malloc_size_of_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e37c5d4cd9473c5f4c9c111f033f15d4df9bd378fdf615944e360a4f55a05f0b" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" -"checksum memory-db 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)" = "828bdf600636e90c56652689f7c3823ae2072104e4b0b5e83ea984f592f12ab9" +"checksum memory-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)" = "881736a0f68a6fae1b596bb066c5bd16d7b3ed645a4dd8ffaefd02f585abaf71" "checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" "checksum merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "de2d16d3b15fec5943d1144f861f61f279d165fdd60998ca262913b9bf1c8adb" "checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" @@ -1406,7 +1406,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" "checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" -"checksum rustc-hex 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "403bb3a286107a04825a5f82e1270acc1e14028d3d554d7a1e08914549575ab8" +"checksum rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" @@ -1428,7 +1428,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c5676413eaeb1ea35300a0224416f57abc3bd251657e0fafc12c47ff98c060" "checksum tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" "checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" -"checksum trie-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)" = "191fda5d0106f3ed35a8c6875428b213e15c516e48129cc263dd7ad16e9a665f" +"checksum trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a0d747ae5b6f078df7e46477fcc7df66df9eb4f27a031cf4a7c890a8dd03d8e6" "checksum trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0b779f7c1c8fe9276365d9d5be5c4b5adeacf545117bb3f64c974305789c5c0b" "checksum twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" "checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 93792e9577b69..1b9532037065c 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -140,7 +140,7 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( // ignore temporary values (values that have null value at the end of operation // AND are not in storage at the beginning of operation if let Some(sk) = storage_key.as_ref() { - if !changes.child_storage_n(sk, k).map(|v| v.0.is_some()).unwrap_or_default() { + if !changes.child_storage_once(sk, k).map(|v| v.0.is_some()).unwrap_or_default() { if let Some(child_info) = child_info { if !backend.exists_child_storage(sk, child_info, k) .map_err(|e| format!("{}", e))? { @@ -149,9 +149,9 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( } } } else { - // we use `storage_n` as this is called only a limited number of time + // we use `storage_once` because this is called a limited number of time // per block processing. - if !changes.storage_n(k).map(|v| v.0.is_some()).unwrap_or_default() { + if !changes.storage_once(k).map(|v| v.0.is_some()).unwrap_or_default() { if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { return Ok(map); } diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index f1700ce26a847..006c41e285375 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -256,7 +256,9 @@ pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber>( // prepare configuration range - we already know zero block. Current block may be the end block if configuration // has been changed in this block - let is_config_changed = match changes.storage_n(sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG) { + let is_config_changed = match changes.storage_once( + sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG + ) { Some((Some(new_config), _)) => new_config != &state.config.encode()[..], Some((None, _)) => true, None => false, diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index 2f191f3376c1a..b234162178bd9 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -431,10 +431,11 @@ impl OverlayedChanges { self.collect_extrinsics = collect_extrinsics; } - /// Same as `storage` but to use carefully as it does not clean - /// tail values and complexity remain the same for following calls. - /// The return boolean indicates if a tail value iteration was required - pub fn storage_n(&self, key: &[u8]) -> Option<(Option<&[u8]>, bool)> { + /// Same as `child_storage` but does not update value. It should + /// consequently be used only when the query is bounded to a bounded + /// number of call before memory get drop. + /// There returned boolean indicates if there is a query overhead. + pub fn storage_once(&self, key: &[u8]) -> Option<(Option<&[u8]>, bool)> { if let Some(overlay_value) = self.changes.top.get(key) { if let (Some(o_value), tail) = overlay_value.get(self.changes.states.as_ref()) { return Some((o_value.value.as_ref().map(|v| v.as_slice()), tail)) @@ -443,9 +444,15 @@ impl OverlayedChanges { None } - /// Same as `child_storage` but to use carefully as it does not clean - /// tail values and complexity remain the same for following calls. - pub fn child_storage_n(&self, storage_key: &[u8], key: &[u8]) -> Option<(Option<&[u8]>, bool)> { + /// Same as `child_storage` but does not update value. It should + /// consequently be used only when the query is bounded to a bounded + /// number of call before memory get drop. + /// There returned boolean indicates if there is a query overhead. + pub fn child_storage_once( + &self, + storage_key: &[u8], + key: &[u8], + ) -> Option<(Option<&[u8]>, bool)> { if let Some(map) = self.changes.children.get(storage_key) { if let Some(overlay_value) = map.0.get(key) { if let (Some(o_value), tail) = overlay_value.get(self.changes.states.as_ref()) { @@ -743,8 +750,6 @@ impl OverlayedChanges { fn extrinsic_index(&mut self) -> Option { match self.collect_extrinsics { true => Some( - // this call to storage is a map access for something that is just - // a counter and should be a field of the struct TODO self.storage(EXTRINSIC_INDEX) .and_then(|idx| idx.and_then(|idx| Decode::decode(&mut &*idx).ok())) .unwrap_or(NO_EXTRINSIC_INDEX)), @@ -762,9 +767,8 @@ impl OverlayedChanges { } #[cfg(any(test, feature = "test-helpers"))] - /// Count the number of key value pairs, at every latest history. - /// Should only be use for debugging or testing, this is slow - /// and technical. + /// Count the number of pending key value pairs. + /// This is slow and should only be use for debugging or testing. pub fn top_count_keyvalue_pair(&self) -> usize { let mut result = 0; for (_, v) in self.changes.top.iter() { @@ -773,9 +777,11 @@ impl OverlayedChanges { result } - /// costy garbage collection of unneeded memory from - /// key values. Eager set to true will remove more - /// key value but allows more costy memory changes. + /// Garbage collection of unneeded memory from + /// every `Layers` of values in regard to current transactional state. + /// If `eager` parameter is true, the garbage collection will go past + /// the latest discarded and committed transaction and reclaim the maximal + /// size of memory at an additional computational cost. pub fn gc(&mut self, eager: bool) { self.changes.gc(eager); } diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index 35b70b3faba96..00dc9e69796c4 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -18,14 +18,15 @@ #[derive(Debug, Clone, Eq, PartialEq, Copy)] -/// State of a transactional layer. +/// Possible states of a transactional layer. pub(crate) enum TransactionState { /// Data is under change and can still be dropped. Pending, - /// Same as pending but does count as a transaction start. + /// Same as `Pending` but does count as a transaction start. TxPending, - /// Data pointing to this indexed historic state should - /// not be returned and can be removed. + /// The transaction has been discarded. + /// Data from a `LayerEntry` pointing to this layer state should + /// not be returned and can be remove. Dropped, } @@ -34,19 +35,17 @@ pub(crate) enum TransactionState { #[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] pub struct States { history: Vec, - // Keep track of the transaction positions. - // This is redundant with `history`. - // This is only use to precalculate the result - // of a backward iteration to find the start - // of a transactional window (function - // `transaction_start_windows`). - // - // At many place we could use this instead - // of the `TxPending` state, but `TxPending` - // state is favored, except for `start_windows` - // case. + // Keep track of the latest start of transaction + // for any `LayerEntry`. + // This information is redundant as it could be + // calculated by iterating backward over `history` + // field. + // Managing this cache allow use to have + // an implementation of `transaction_start_windows` + // function that is constant time. start_transaction_window: Vec, - // commit is the position in history of the latest committed. + // `commit` stores the index of the layer for which a previous + // prospective commit occurs. commit: usize, } @@ -61,15 +60,14 @@ impl Default for States { } impl States { - /// Get reference of state, that is enough - /// information to query historical - /// data. + /// Get reference of states, this is enough + /// information to query a `Layers` of values. pub(crate) fn as_ref(&self) -> &[TransactionState] { self.history.as_ref() } /// Current number of inner states. - pub fn num_states(&self) -> usize { + pub(crate) fn num_states(&self) -> usize { self.history.len() } @@ -82,14 +80,16 @@ impl States { /// index. /// This can only work if there was no eager /// garbage collection (eager collection can - /// change interval between `old_committed` and - /// `self.commit`). + /// drop `LayersEntry` in the interval between + /// `old_committed` and `self.commit` and moving + /// back the committed cursor is therefore not + /// allowed in this case). pub fn unchecked_rollback_committed(&mut self, old_committed: usize) { self.commit = old_committed; self.discard_prospective(); } - /// Build any state for testing only. + /// Instantiate a random state, only for testing. #[cfg(test)] pub(crate) fn test_vector( history: Vec, @@ -99,8 +99,8 @@ impl States { States { history, start_transaction_window, commit } } - /// Discard prospective changes to state. - /// That is revert all transaction up to the committed index. + /// Discard prospective changes. + /// This action invalidates all transaction state up to the committed index. pub fn discard_prospective(&mut self) { for i in self.commit .. self.history.len() { self.history[i] = TransactionState::Dropped; @@ -110,7 +110,10 @@ impl States { self.start_transaction_window.push(self.commit); } - /// Commit prospective changes to state. + /// Commit prospective changes. + /// It push the committed pointer at the very + /// end of the state, blocking discard of any + /// current state. pub fn commit_prospective(&mut self) { self.commit = self.history.len(); self.history.push(TransactionState::Pending); @@ -127,7 +130,8 @@ impl States { } /// Discard a transactional layer. - /// A transaction is always running (history always end with pending). + /// A transaction is always running, so this also append + /// a new pending transactional layer to the state. pub fn discard_transaction(&mut self) { let mut i = self.history.len(); while i > self.commit { @@ -153,6 +157,11 @@ impl States { /// Commit a transactional layer. pub fn commit_transaction(&mut self) { let mut i = self.history.len(); +/* TODO test and fuzz with this implementation (replaces while loop). + let latest_transaction = self.start_transaction_window[i - 1]; + if latest_transaction > self.commit { + self.history[latest_transaction] = TransactionState::Pending; + } */ while i > self.commit { i -= 1; match self.history[i] { @@ -164,8 +173,10 @@ impl States { }, } } + // TODO test without this push self.history.push(TransactionState::Pending); self.start_transaction_window.truncate(i); + // self.start_transaction_window.truncate(latest_transaction); let previous = self.start_transaction_window.last() .cloned().unwrap_or(self.commit); self.start_transaction_window.resize(self.history.len(), previous); @@ -300,6 +311,7 @@ impl Layers { match states.history[state_index] { TransactionState::TxPending => { if state_index >= start_transaction_window { + debug_assert!(state_index == start_transaction_window); // TODO test replace this condition previous_switch = Some(index); } else { if result.is_none() { @@ -314,6 +326,8 @@ impl Layers { } else { if result.is_none() { result = Some((index, state_index)); + // continue iteration to find all `LayersEntry` from this + // start_transaction_window. start_transaction_window = transaction_start_windows(states, state_index); } else { break; @@ -352,6 +366,9 @@ impl Layers { | state @ TransactionState::Pending => { if state_index < states.commit && index > prune_index { prune_index = index; + // TODO break ?? and remove condition?? and later in break + // condition do not if state_index < states.commit, just do + // not break!! } if state_index >= start_transaction_window { @@ -377,6 +394,7 @@ impl Layers { } } let deleted = if prune_to_commit && prune_index > 0 && result.is_some() { + // Remove values before first value needed to read committed state. self.remove_until(prune_index); prune_index } else { @@ -396,19 +414,23 @@ impl Layers { // not needed (before another committed value). // `previous_switch` is the index to the first unneeded value. // - // An index offset is here in case some content was `deleted` before - // this function call. + // An index offset is here in case some content was `deleted` from + // the `Layers` head without updating the input indexes. fn drop_last_values( &mut self, result: Option<(usize, usize)>, previous_switch: Option, + // TODO remove it from result before call. deleted: usize, ) -> Option> { if let Some((index, state_index)) = result { + // Remove terminal values. if index + 1 - deleted < self.0.len() { self.0.truncate(index + 1 - deleted); } if let Some(switch_index) = previous_switch { + // We need to remove some value in between + // transaction start and this latest value. if let Some(mut value) = self.0.pop() { self.0.truncate(switch_index - deleted); value.index = state_index; @@ -511,6 +533,9 @@ impl Layers { /// A default sample configuration to manage garbage collection /// triggering. +/// +/// With this default values it should be very unlikelly that gc is needed +/// during a block processing for most use case. pub(crate) const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { trigger_transaction_gc: 1_000_000, trigger_commit_gc: 100_000, @@ -518,16 +543,17 @@ pub(crate) const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { }; /// Garbage collection configuration. -/// It is designed to listen on two different operation, transaction -/// and commit. -/// It match transcational semantic with transaction for transaction -/// operation and commit for prospective operation. -/// Considering transaction can be manipulated in extrinsic, this option -/// is quite unsafe and shoudl be use with care (the default value is -/// explicitelly very high). -/// More generally it should be very unlikelly that gc is needed -/// during a block processing for most use case. +/// It is designed to listen on two different event, transaction +/// or commit. +/// Transaction operations are `discard_transaction` and `commit_transaction`. +/// Prospective operations are `discard_prospective` and `commit_prospective`. +/// +/// Transaction can be call at any time, while prospective operation are only +/// call between extrinsics, therefore transaction treshold should be higher +/// than commit for being more frequent. /// +/// It measures operation, that is write or delete storage, giving an idea +/// of a possible maximum size of used memory. /// This is not a very good measurement and should be replace by /// heap measurement or other metrics, at this point the gc even /// if tested should not be required. From 5f268696df1fa52a77aaa7acf06f7c83d8a12f47 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 20 Jan 2020 19:45:16 +0100 Subject: [PATCH 120/129] best loop condition, and manage deleted offset out of actual deletion. --- .../state-machine/src/transaction_layers.rs | 79 +++++++------------ 1 file changed, 28 insertions(+), 51 deletions(-) diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index 00dc9e69796c4..ea46c823e00a4 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -156,26 +156,12 @@ impl States { /// Commit a transactional layer. pub fn commit_transaction(&mut self) { - let mut i = self.history.len(); -/* TODO test and fuzz with this implementation (replaces while loop). - let latest_transaction = self.start_transaction_window[i - 1]; + let latest_transaction = self.start_transaction_window[self.history.len() - 1]; if latest_transaction > self.commit { self.history[latest_transaction] = TransactionState::Pending; - } */ - while i > self.commit { - i -= 1; - match self.history[i] { - TransactionState::Pending - | TransactionState::Dropped => (), - TransactionState::TxPending => { - self.history[i] = TransactionState::Pending; - break; - }, - } } - // TODO test without this push self.history.push(TransactionState::Pending); - self.start_transaction_window.truncate(i); + self.start_transaction_window.truncate(latest_transaction); // self.start_transaction_window.truncate(latest_transaction); let previous = self.start_transaction_window.last() .cloned().unwrap_or(self.commit); @@ -311,11 +297,10 @@ impl Layers { match states.history[state_index] { TransactionState::TxPending => { if state_index >= start_transaction_window { - debug_assert!(state_index == start_transaction_window); // TODO test replace this condition previous_switch = Some(index); } else { if result.is_none() { - result = Some((index, state_index)); + result = Some((index + 1, state_index)); } } break; @@ -325,7 +310,7 @@ impl Layers { previous_switch = Some(index); } else { if result.is_none() { - result = Some((index, state_index)); + result = Some((index + 1, state_index)); // continue iteration to find all `LayersEntry` from this // start_transaction_window. start_transaction_window = transaction_start_windows(states, state_index); @@ -337,7 +322,7 @@ impl Layers { TransactionState::Dropped => (), } } - self.drop_last_values(result, previous_switch, 0) + self.drop_last_values(result, previous_switch) } /// Method to prune regarding a full state. @@ -354,7 +339,7 @@ impl Layers { if self_len == 0 { return None; } - let mut prune_index = 0; + let mut head_prune_index = 0; debug_assert!(states.history.len() >= self_len); let mut result = None; let mut start_transaction_window = usize::max_value(); @@ -364,43 +349,37 @@ impl Layers { match states.history[state_index] { state @ TransactionState::TxPending | state @ TransactionState::Pending => { - if state_index < states.commit && index > prune_index { - prune_index = index; - // TODO break ?? and remove condition?? and later in break - // condition do not if state_index < states.commit, just do - // not break!! - } - if state_index >= start_transaction_window { previous_switch = Some(index); } else { if result.is_none() { - result = Some((index, state_index)); + result = Some((index + 1, state_index)); if state == TransactionState::Pending { start_transaction_window = transaction_start_windows(states, state_index); } } else { - if prune_to_commit { - if state_index < states.commit { - break; - } - } else { + if !prune_to_commit { break; } } } + + if state_index < states.commit { + head_prune_index = index; + break; + } }, TransactionState::Dropped => (), } } - let deleted = if prune_to_commit && prune_index > 0 && result.is_some() { + if prune_to_commit && head_prune_index > 0 && result.is_some() { // Remove values before first value needed to read committed state. - self.remove_until(prune_index); - prune_index - } else { - 0 - }; - self.drop_last_values(result, previous_switch, deleted) + self.remove_until(head_prune_index); + previous_switch = previous_switch.map(|switch| switch - head_prune_index); + result = result + .map(|(index, state_index)| (index - head_prune_index, state_index)); + } + self.drop_last_values(result, previous_switch) } // Function used to remove all last values for `get_mut` and @@ -413,32 +392,27 @@ impl Layers { // It also does remove values on committed state that are // not needed (before another committed value). // `previous_switch` is the index to the first unneeded value. - // - // An index offset is here in case some content was `deleted` from - // the `Layers` head without updating the input indexes. fn drop_last_values( &mut self, result: Option<(usize, usize)>, previous_switch: Option, - // TODO remove it from result before call. - deleted: usize, ) -> Option> { if let Some((index, state_index)) = result { // Remove terminal values. - if index + 1 - deleted < self.0.len() { - self.0.truncate(index + 1 - deleted); + if index < self.0.len() { + self.0.truncate(index); } if let Some(switch_index) = previous_switch { // We need to remove some value in between // transaction start and this latest value. if let Some(mut value) = self.0.pop() { - self.0.truncate(switch_index - deleted); + self.0.truncate(switch_index); value.index = state_index; self.push_unchecked(value); } - Some((&mut self.0[switch_index - deleted].value, state_index).into()) + Some((&mut self.0[switch_index].value, state_index).into()) } else { - Some((&mut self.0[index - deleted].value, state_index).into()) + Some((&mut self.0[index - 1].value, state_index).into()) } } else { self.0.clear(); @@ -834,6 +808,9 @@ pub mod fuzz { vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], + vec![0xff,0xd,0x0,0x61,0x8,0x9c,0xff,0x57,0xd3,0xff,0xd1,0xd1,0x26, + 0xff,0x33,0xff,0x24,0x1f,0xff,0xff,0xdd,0x0,0x8,0x7a,0x7f,0xff,0xff, + 0x26,0xff,0x7b,0xff,0xff,0x26,0xee,0xff,0xff,0x41,0x83], ]; for input in inputs.iter() { fuzz_transactions_inner(&input[..], true); From 09539d443612fe0ca62044cb622fae2bbdc2804c Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 21 Jan 2020 09:21:15 +0100 Subject: [PATCH 121/129] fix benches. --- .../state-machine/src/overlayed_changes.rs | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index b234162178bd9..9ee7301adc738 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -492,7 +492,7 @@ impl OverlayedChanges { /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_storage(&mut self, key: StorageKey, value: Option) { + pub fn set_storage(&mut self, key: StorageKey, value: Option) { self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); let entry = self.changes.top.entry(key).or_default(); @@ -507,7 +507,7 @@ impl OverlayedChanges { /// Inserts the given key-value pair into the prospective child change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_child_storage( + pub fn set_child_storage( &mut self, storage_key: StorageKey, child_info: ChildInfo, @@ -533,10 +533,12 @@ impl OverlayedChanges { /// Clear child storage of given storage key. /// /// NOTE that this doesn't take place immediately but written into the prospective - /// change set, and still can be reverted by [`discard_prospective`]. + /// change set, and still can be reverted by [`discard_prospective`] + /// or [`discard_transaction`] /// /// [`discard_prospective`]: #method.discard_prospective - pub(crate) fn clear_child_storage( + /// [`discard_transaction`]: #method.discard_transaction + pub fn clear_child_storage( &mut self, storage_key: &[u8], child_info: ChildInfo, @@ -556,10 +558,12 @@ impl OverlayedChanges { /// Removes all key-value pairs which keys share the given prefix. /// /// NOTE that this doesn't take place immediately but written into the prospective - /// change set, and still can be reverted by [`discard_prospective`]. + /// change set, and still can be reverted by [`discard_prospective`] + /// or [`discard_transaction`] /// /// [`discard_prospective`]: #method.discard_prospective - pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { + /// [`discard_transaction`]: #method.discard_transaction + pub fn clear_prefix(&mut self, prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); let mut number_removed = 0; @@ -572,7 +576,16 @@ impl OverlayedChanges { self.operation_from_last_gc += number_removed; } - pub(crate) fn clear_child_prefix( + /// Removes all key-value pairs which keys share the given prefix for + /// a given child trie. + /// + /// NOTE that this doesn't take place immediately but written into the prospective + /// change set, and still can be reverted by [`discard_prospective`] + /// or [`discard_transaction`] + /// + /// [`discard_prospective`]: #method.discard_prospective + /// [`discard_transaction`]: #method.discard_transaction + pub fn clear_child_prefix( &mut self, storage_key: &[u8], child_info: ChildInfo, From b109130b17fceb27dc606155a89af421a0a49a99 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 30 Jan 2020 18:28:00 +0100 Subject: [PATCH 122/129] Fixing line lengths. --- frame/support/src/storage/mod.rs | 6 +-- primitives/io/src/lib.rs | 2 +- primitives/state-machine/src/ext.rs | 15 ++++--- .../state-machine/src/transaction_layers.rs | 45 ++++++++++--------- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index 5e6c673ed48e9..6fdab4547a1f9 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -32,11 +32,11 @@ pub mod generator; /// the transactional layer get reverted; otherwise /// it is committed. pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { - sp_io::transaction::start_transaction(); + sp_io::storage_transaction::start_transaction(); let result = f(); match result { - Ok(_) => sp_io::transaction::commit_transaction(), - Err(_) => sp_io::transaction::discard_transaction(), + Ok(_) => sp_io::storage_transaction::commit_transaction(), + Err(_) => sp_io::storage_transaction::discard_transaction(), } result } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index a16776559132b..ca731d699c525 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -315,7 +315,7 @@ pub trait Storage { /// Interface for managing transaction within the runtime. #[runtime_interface] -pub trait Transaction { +pub trait StorageTransaction { /// Start a new transaction. fn start_transaction(&mut self) { self.storage_start_transaction(); diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 98470a31b0201..a5aae736c7acf 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -136,7 +136,10 @@ where self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.changes.iter_values(None).map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec())))) + .chain( + self.overlay.changes.iter_values(None) + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))) + ) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -338,12 +341,12 @@ where let next_overlay_key_change = self.overlay.next_storage_key_change(key); match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => return Some(backend_key), + (Some(backend_key), Some(overlay_key)) + if &backend_key[..] < overlay_key.0 => return Some(backend_key), (backend_key, None) => return backend_key, (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { return Some(overlay_key.0.to_vec()) } else { - // TODO make this function non recursive to avoid this clone overlay_key.0.to_vec() }, } @@ -367,7 +370,8 @@ where ); match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => return Some(backend_key), + (Some(backend_key), Some(overlay_key)) + if &backend_key[..] < overlay_key.0 => return Some(backend_key), (backend_key, None) => return backend_key, (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { return Some(overlay_key.0.to_vec()) @@ -376,9 +380,6 @@ where }, } }; - // TODO no need to query child at each recursive iter here - // and also find a way to remove the clone (non recursive and global mut handle should do the - // trick). self.next_child_storage_key(storage_key, child_info, &next_key[..]) } diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index ea46c823e00a4..cc0fe70240dc8 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Types and method for managing a stack of transactional values. +//! Types and methods for managing a stack of transactional values. #[derive(Debug, Clone, Eq, PartialEq, Copy)] @@ -26,7 +26,7 @@ pub(crate) enum TransactionState { TxPending, /// The transaction has been discarded. /// Data from a `LayerEntry` pointing to this layer state should - /// not be returned and can be remove. + /// not be returned and can be removed. Dropped, } @@ -40,9 +40,8 @@ pub struct States { // This information is redundant as it could be // calculated by iterating backward over `history` // field. - // Managing this cache allow use to have - // an implementation of `transaction_start_windows` - // function that is constant time. + // Managing this cache let us implement `transaction_start_windows` + // function in constant time. start_transaction_window: Vec, // `commit` stores the index of the layer for which a previous // prospective commit occurs. @@ -248,8 +247,7 @@ impl Layers { None } - /// Push a value without checking without transactional layer - /// consistency. + /// Push a value without checking transactional layer consistency. pub(crate) fn push_unchecked(&mut self, item: LayerEntry) { self.0.push(item); } @@ -355,7 +353,10 @@ impl Layers { if result.is_none() { result = Some((index + 1, state_index)); if state == TransactionState::Pending { - start_transaction_window = transaction_start_windows(states, state_index); + start_transaction_window = transaction_start_windows( + states, + state_index, + ); } } else { if !prune_to_commit { @@ -459,6 +460,14 @@ impl Layers { } } +#[cfg(any(test, feature = "test-helpers"))] +impl Layers { + /// Get current number of stored historical values. + pub fn len(&self) -> usize { + self.0.len() + } +} + #[cfg(test)] impl Layers { /// Create an history from a sequence of historical values. @@ -473,7 +482,7 @@ impl Layers { /// Get latest prospective value, excludes /// committed values. pub(crate) fn get_prospective(&self, states: &States) -> Option<&V> { - let self_len = self.len(); + let self_len = self.0.len(); if self_len == 0 { return None; } @@ -488,7 +497,7 @@ impl Layers { /// Get latest committed value. pub(crate) fn get_committed(&self, states: &States) -> Option<&V> { - let self_len = self.len(); + let self_len = self.0.len(); if self_len == 0 { return None; } @@ -504,12 +513,11 @@ impl Layers { } } - /// A default sample configuration to manage garbage collection /// triggering. /// /// With this default values it should be very unlikelly that gc is needed -/// during a block processing for most use case. +/// during a block processing for most use cases. pub(crate) const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { trigger_transaction_gc: 1_000_000, trigger_commit_gc: 100_000, @@ -566,6 +574,7 @@ impl GCConfiguration { } } + /// Base code for fuzzing. #[cfg(any(test, feature = "test-helpers"))] pub mod fuzz { @@ -579,13 +588,6 @@ pub mod fuzz { /// Size of key, max 255 const VALUE_SPACE: u8 = 50; - impl Layers { - /// Get current number of stored historical values. - pub fn len(&self) -> usize { - self.0.len() - } - } - /// Fuzz input against a stack of hash map reference implementation. /// `check_compactness` add a check in the number of stored entry. pub fn fuzz_transactions_inner(input: &[u8], check_compactness: bool) { @@ -671,7 +673,10 @@ pub mod fuzz { assert!(success); } - fn check_values(overlayed: &OverlayedChanges, ref_overlayed: &RefOverlayedChanges) -> (bool, usize) { + fn check_values( + overlayed: &OverlayedChanges, + ref_overlayed: &RefOverlayedChanges, + ) -> (bool, usize) { let mut len = 0; let mut success = true; for (key, value) in overlayed.iter_values(None) { From 1db0bec86105d836f1552fe15edff565728e62f0 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 30 Jan 2020 18:33:45 +0100 Subject: [PATCH 123/129] increment spec version --- bin/node/runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index aa550893f93a5..2e4d894b49888 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -80,7 +80,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 210, + spec_version: 211, impl_version: 0, apis: RUNTIME_API_VERSIONS, }; From 121e64be62c709fca976502c8e9e38eec8473b9e Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 30 Jan 2020 18:37:56 +0100 Subject: [PATCH 124/129] storage transaction host module name. --- primitives/io/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index c6b8cd15e8def..3b4895d97ac38 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -935,7 +935,7 @@ pub type TestExternalities = sp_state_machine::TestExternalities Date: Thu, 30 Jan 2020 18:47:41 +0100 Subject: [PATCH 125/129] 3 new function, switching index of missing one in wasmtime test. --- client/executor/src/integration_tests/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 3f3d9f69e13dc..949dcdd91c6cd 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -117,7 +117,7 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) { #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"call to undefined external function with index 69\")" + "Other(\"call to undefined external function with index 72\")" ), } } From 46e54c870d9d71efda79116a90b94d9c0c4466e9 Mon Sep 17 00:00:00 2001 From: Emeric Chevalier Date: Thu, 30 Jan 2020 18:53:23 +0100 Subject: [PATCH 126/129] another hardcoded wasmtime index in test. --- client/executor/src/integration_tests/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 949dcdd91c6cd..2ac67215a82f0 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -88,7 +88,7 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) { #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"call to undefined external function with index 68\")" + "Other(\"call to undefined external function with index 71\")" ), } } From 4d6af74a8cc27f7bfea3d8de8d789c8640e79d49 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 2 Mar 2020 12:09:03 +0100 Subject: [PATCH 127/129] Added a descriptive test of internal state. This does not intend to cover all case but is a good entry point to read this PR. --- .../state-machine/src/transaction_layers.rs | 134 +++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs index e46d9c6ea0c10..547a6edf69765 100644 --- a/primitives/state-machine/src/transaction_layers.rs +++ b/primitives/state-machine/src/transaction_layers.rs @@ -179,7 +179,7 @@ pub fn transaction_start_windows(states: &States, from: usize) -> usize { } /// Stack of values at different transactional layers. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); #[cfg(test)] @@ -198,7 +198,7 @@ impl Default for Layers { } /// An value entry of a indexed transactional layer. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct LayerEntry { /// The stored value. pub value: V, @@ -820,4 +820,134 @@ pub mod fuzz { fuzz_transactions_inner(&input[..], true); } } + + #[test] + fn test_internal_state() { + use super::{States, Layers, TransactionState}; + + let check_state = |global_state: &States, global_state_ref: &States, layers: &[&Layers], layers_ref: &[&[(usize, usize)]]| { + let layers_ref: Vec<_> = layers_ref.iter().map(|internal| + Layers::from_iter(internal.iter().map(Clone::clone).map(Into::into).into_iter()) + ).collect(); + let layers_ref_ref: Vec<_> = layers_ref.iter().collect(); + assert_eq!(global_state, global_state_ref); + assert_eq!(layers, &layers_ref_ref[..]); + }; + + let mut global_state = States::default(); + let ref_state = States::test_vector( + vec![TransactionState::Pending], + vec![0], + 0, + ); + let mut value1 = Layers::default(); + check_state(&global_state, &ref_state, &[&value1], &[&[]]); + value1.set(&global_state, 11); + check_state(&global_state, &ref_state, &[&value1], &[&[(11, 0)]]); + value1.set(&global_state, 12); + let mut value2 = Layers::default(); + check_state(&global_state, &ref_state, &[&value1, &value2], &[&[(12, 0)], &[]]); + global_state.start_transaction(); + let ref_state = States::test_vector( + vec![TransactionState::Pending, TransactionState::TxPending], + vec![0, 1], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], &[&[(12, 0)], &[]]); + value2.set(&global_state, 11); + let values_ref = &[ + &[(12, 0)][..], + &[(11, 1)][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + value1.set(&global_state, 13); + let values_ref = &[ + &[(12, 0), (13, 1)][..], + &[(11, 1)][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.start_transaction(); + let ref_state = States::test_vector( + vec![TransactionState::Pending, TransactionState::TxPending, TransactionState::TxPending], + vec![0, 1, 2], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.discard_transaction(); + let ref_state = States::test_vector( + vec![TransactionState::Pending, TransactionState::TxPending, TransactionState::Dropped, TransactionState::Pending], + vec![0, 1, 1, 1], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + value2.set(&global_state, 12); + let values_ref = &[ + &[(12, 0), (13, 1)][..], + &[(11, 1), (12, 3)][..], + ][..]; + assert_eq!(value1.get(global_state.as_ref()), (Some(&13), false)); + assert_eq!(value2.get(global_state.as_ref()), (Some(&12), false)); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.discard_transaction(); + let ref_state = States::test_vector( + vec![ + TransactionState::Pending, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Pending, + ], + vec![0, 0, 0, 0, 0], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + assert_eq!(value1.get(global_state.as_ref()), (Some(&12), true)); + assert_eq!(value2.get(global_state.as_ref()), (None, true)); + // update value 1 & value 2 by using get_mut. + assert!(value1.get_mut(&global_state).is_some()); + assert!(value2.get_mut(&global_state).is_none()); + assert_eq!(value1.get(global_state.as_ref()), (Some(&12), false)); + assert_eq!(value2.get(global_state.as_ref()), (None, false)); + let values_ref = &[ + &[(12, 0)][..], + &[][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.start_transaction(); + value2.set(&global_state, 14); + let ref_state = States::test_vector( + vec![ + TransactionState::Pending, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Pending, + TransactionState::TxPending, + ], + vec![0, 0, 0, 0, 0, 5], + 0, + ); + let values_ref = &[ + &[(12, 0)][..], + &[(14, 5)][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.commit_prospective(); + let ref_state = States::test_vector( + vec![ + TransactionState::Pending, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Pending, + TransactionState::TxPending, + TransactionState::Pending, + ], + vec![6, 6, 6, 6, 6, 6, 6], + 6, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + assert_eq!(value1.get(global_state.as_ref()), (Some(&12), false)); + assert_eq!(value2.get(global_state.as_ref()), (Some(&14), false)); + } } From 683abcb6ce38907473b38d9e3ee9764642a1ceaa Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 10 Mar 2020 15:38:56 +0100 Subject: [PATCH 128/129] update fuzzer cargo.lock --- primitives/state-machine/fuzz/Cargo.lock | 1164 +++++++++++----------- 1 file changed, 590 insertions(+), 574 deletions(-) diff --git a/primitives/state-machine/fuzz/Cargo.lock b/primitives/state-machine/fuzz/Cargo.lock index fd8a67dc7947d..c7ff1e839534a 100644 --- a/primitives/state-machine/fuzz/Cargo.lock +++ b/primitives/state-machine/fuzz/Cargo.lock @@ -4,1441 +4,1457 @@ name = "Inflector" version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" dependencies = [ - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static", + "regex", ] [[package]] name = "ahash" version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" dependencies = [ - "const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "const-random", ] [[package]] name = "aho-corasick" -version = "0.7.6" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" dependencies = [ - "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr", ] [[package]] name = "arbitrary" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" [[package]] name = "arrayref" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" [[package]] name = "arrayvec" version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" dependencies = [ - "nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "nodrop", ] [[package]] name = "arrayvec" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" [[package]] name = "autocfg" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "backtrace" -version = "0.3.40" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" dependencies = [ - "backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace-sys", + "cfg-if", + "libc", + "rustc-demangle", ] [[package]] name = "backtrace-sys" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17b52e737c40a7d75abca20b29a19a0eb7ba9fc72c5a72dd282a0a3c2c0dc35" dependencies = [ - "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", + "cc", + "libc", ] [[package]] name = "base58" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" [[package]] name = "bitflags" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "bitvec" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" [[package]] name = "blake2-rfc" version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" dependencies = [ - "arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.4.12", + "constant_time_eq", ] [[package]] name = "block-buffer" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" dependencies = [ - "block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "block-padding", + "byte-tools", + "byteorder", + "generic-array", ] [[package]] name = "block-padding" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" dependencies = [ - "byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byte-tools", ] [[package]] name = "byte-slice-cast" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" [[package]] name = "byte-tools" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "byteorder" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "c2-chacha" -version = "0.2.3" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" [[package]] name = "cc" -version = "1.0.47" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" [[package]] name = "cfg-if" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] name = "clear_on_drop" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" dependencies = [ - "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", + "cc", ] [[package]] name = "cloudabi" version = "0.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" dependencies = [ - "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags", ] [[package]] name = "const-random" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" dependencies = [ - "const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", + "const-random-macro", + "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" dependencies = [ - "proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom", + "proc-macro-hack", ] [[package]] name = "constant_time_eq" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "crunchy" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" [[package]] name = "crypto-mac" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" dependencies = [ - "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", - "subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array", + "subtle 1.0.0", ] [[package]] name = "curve25519-dalek" version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", + "clear_on_drop", + "digest", + "rand_core 0.3.1", + "subtle 2.2.2", ] [[package]] name = "curve25519-dalek" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", + "digest", + "rand_core 0.5.1", + "subtle 2.2.2", + "zeroize 1.1.0", ] [[package]] name = "digest" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" dependencies = [ - "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "generic-array", ] [[package]] name = "ed25519-dalek" version = "1.0.0-pre.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" dependencies = [ - "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "curve25519-dalek 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "clear_on_drop", + "curve25519-dalek 2.0.0", + "rand 0.7.3", + "sha2", ] [[package]] name = "environmental" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" [[package]] name = "failure" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" dependencies = [ - "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace", + "failure_derive", ] [[package]] name = "failure_derive" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "syn", + "synstructure", ] [[package]] name = "fake-simd" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "fixed-hash" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3367952ceb191f4ab95dd5685dc163ac539e36202f9fcfd0cb22f9f9c542fefc" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", + "libc", + "rand 0.7.3", + "rustc-hex", + "static_assertions", ] [[package]] name = "fuchsia-cprng" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" [[package]] name = "generic-array" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" dependencies = [ - "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "typenum", ] [[package]] name = "getrandom" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if", + "libc", + "wasi", ] [[package]] name = "hash-db" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" [[package]] name = "hash256-std-hasher" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" dependencies = [ - "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "hashbrown" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy", ] [[package]] name = "hashbrown" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" dependencies = [ - "ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "ahash", + "autocfg 0.1.7", ] [[package]] name = "hex" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" [[package]] name = "hmac" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" dependencies = [ - "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crypto-mac", + "digest", ] [[package]] name = "hmac-drbg" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" dependencies = [ - "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", - "generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", - "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "digest", + "generic-array", + "hmac", ] [[package]] name = "impl-codec" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" dependencies = [ - "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec", ] [[package]] name = "impl-serde" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e3cae7e99c7ff5a995da2cf78dd0a5383740eda71d98cf7b1910c301ac69b8" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-serde" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bbe9ea9b182f0fb1cabbd61f4ff9b7b7b9197955e95a7e4c27de5055eb29ff8" dependencies = [ - "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", ] [[package]] name = "impl-trait-for-tuples" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "keccak" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" [[package]] name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.65" +version = "0.2.67" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" [[package]] name = "libfuzzer-sys" version = "0.1.0" -source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#4ad88ec54fa2e7a0c877b6477cc86ea0de14a90d" +source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#35ce7d7177c254b9c798aec171dfe76877d1a20f" dependencies = [ - "arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)", + "arbitrary", + "cc", ] [[package]] name = "libsecp256k1" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hmac-drbg 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "lock_api" -version = "0.1.5" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962" dependencies = [ - "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayref", + "crunchy", + "digest", + "hmac-drbg", + "rand 0.7.3", + "sha2", + "subtle 2.2.2", + "typenum", ] [[package]] name = "lock_api" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" dependencies = [ - "scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "scopeguard", ] [[package]] name = "log" version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "malloc_size_of_derive" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if", ] [[package]] name = "maybe-uninit" version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.2.1" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" [[package]] name = "memory-db" -version = "0.18.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" dependencies = [ - "ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-util-mem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ahash", + "hash-db", + "hashbrown", + "parity-util-mem", ] [[package]] name = "memory_units" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" [[package]] name = "merlin" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0942b357c1b4d0dc43ba724674ec89c3218e6ca2b3e8269e7cb53bcecd2f6e" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", + "keccak", + "rand_core 0.4.2", + "zeroize 1.1.0", ] [[package]] name = "nodrop" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" [[package]] name = "num-bigint" -version = "0.2.3" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0", + "num-integer", + "num-traits", ] [[package]] name = "num-integer" -version = "0.1.41" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0", + "num-traits", ] [[package]] name = "num-rational" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0", + "num-bigint", + "num-integer", + "num-traits", ] [[package]] name = "num-traits" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 1.0.0", ] [[package]] name = "once_cell" -version = "0.1.8" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" dependencies = [ - "parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.9.0", ] [[package]] name = "opaque-debug" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" [[package]] name = "parity-scale-codec" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" dependencies = [ - "arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "bitvec 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "byte-slice-cast 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec-derive 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "arrayvec 0.5.1", + "bitvec", + "byte-slice-cast", + "parity-scale-codec-derive", + "serde", ] [[package]] name = "parity-scale-codec-derive" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" dependencies = [ - "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "parity-util-mem" -version = "0.3.0" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" +dependencies = [ + "cfg-if", + "impl-trait-for-tuples", + "parity-util-mem-derive", + "parking_lot 0.10.0", + "primitive-types", + "winapi", +] + +[[package]] +name = "parity-util-mem-derive" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "malloc_size_of_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "syn", + "synstructure", ] [[package]] name = "parity-wasm" version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" [[package]] name = "parking_lot" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" dependencies = [ - "lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lock_api", + "parking_lot_core 0.6.2", + "rustc_version", ] [[package]] name = "parking_lot" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" dependencies = [ - "lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "lock_api", + "parking_lot_core 0.7.0", ] [[package]] name = "parking_lot_core" -version = "0.4.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec 0.6.13", + "winapi", ] [[package]] name = "parking_lot_core" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" dependencies = [ - "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "smallvec 1.2.0", + "winapi", ] [[package]] name = "pbkdf2" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder", + "crypto-mac", ] [[package]] name = "ppv-lite86" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" [[package]] name = "primitive-types" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4336f4f5d5524fa60bcbd6fe626f9223d8142a50e7053e979acdf0da41ab975" dependencies = [ - "fixed-hash 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "impl-codec 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "uint 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "fixed-hash", + "impl-codec", + "impl-serde 0.3.0", + "uint", ] [[package]] name = "proc-macro-crate" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" dependencies = [ - "toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "toml", ] [[package]] name = "proc-macro-hack" version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "proc-macro2" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" dependencies = [ - "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid", ] [[package]] name = "quote" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", ] [[package]] name = "rand" version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", ] [[package]] name = "rand" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", ] [[package]] name = "rand_chacha" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7", + "rand_core 0.3.1", ] [[package]] name = "rand_chacha" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" dependencies = [ - "c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] name = "rand_core" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" dependencies = [ - "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.4.2", ] [[package]] name = "rand_core" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" [[package]] name = "rand_core" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)", + "getrandom", ] [[package]] name = "rand_hc" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "rand_hc" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" dependencies = [ - "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.5.1", ] [[package]] name = "rand_isaac" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "rand_jitter" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "rand_core 0.4.2", + "winapi", ] [[package]] name = "rand_os" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" dependencies = [ - "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", ] [[package]] name = "rand_pcg" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" dependencies = [ - "autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "autocfg 0.1.7", + "rand_core 0.4.2", ] [[package]] name = "rand_xorshift" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "rdrand" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" dependencies = [ - "rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.3.1", ] [[package]] name = "redox_syscall" version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" [[package]] name = "regex" -version = "1.3.1" +version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" dependencies = [ - "aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", - "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", ] [[package]] name = "regex-syntax" -version = "0.6.12" +version = "0.6.16" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" [[package]] name = "rustc-demangle" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hex" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" [[package]] name = "rustc_version" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "semver", ] [[package]] name = "schnorrkel" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" dependencies = [ - "curve25519-dalek 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "zeroize 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", + "curve25519-dalek 1.2.3", + "failure", + "merlin", + "rand 0.6.5", + "rand_core 0.4.2", + "rand_os", + "sha2", + "subtle 2.2.2", + "zeroize 0.9.3", ] [[package]] name = "scopeguard" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "scopeguard" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "semver" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" dependencies = [ - "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "semver-parser", ] [[package]] name = "semver-parser" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" dependencies = [ - "serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "sha2" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" dependencies = [ - "block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", - "fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", ] [[package]] name = "smallvec" version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" dependencies = [ - "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "maybe-uninit", ] [[package]] name = "smallvec" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" [[package]] name = "sp-core" -version = "2.0.0" -dependencies = [ - "base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "ed25519-dalek 1.0.0-pre.3 (registry+https://github.com/rust-lang/crates.io-index)", - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-debug-derive 2.0.0", - "sp-externalities 0.8.0", - "sp-runtime-interface 2.0.0", - "sp-std 2.0.0", - "sp-storage 2.0.0", - "substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", - "zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +version = "2.0.0-alpha.3" +dependencies = [ + "base58", + "blake2-rfc", + "byteorder", + "ed25519-dalek", + "hash-db", + "hash256-std-hasher", + "hex", + "impl-serde 0.3.0", + "lazy_static", + "libsecp256k1", + "log", + "num-traits", + "parity-scale-codec", + "parity-util-mem", + "parking_lot 0.10.0", + "primitive-types", + "rand 0.7.3", + "regex", + "rustc-hex", + "schnorrkel", + "serde", + "sha2", + "sp-debug-derive", + "sp-externalities", + "sp-runtime-interface", + "sp-std", + "sp-storage", + "substrate-bip39", + "tiny-bip39", + "tiny-keccak", + "twox-hash", + "wasmi", + "zeroize 1.1.0", ] [[package]] name = "sp-debug-derive" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "sp-externalities" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ - "environmental 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-std 2.0.0", - "sp-storage 2.0.0", + "environmental", + "sp-std", + "sp-storage", ] [[package]] name = "sp-panic-handler" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace", + "log", ] [[package]] name = "sp-runtime-interface" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-externalities 0.8.0", - "sp-runtime-interface-proc-macro 2.0.0", - "sp-std 2.0.0", - "sp-wasm-interface 2.0.0", - "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-scale-codec", + "primitive-types", + "sp-externalities", + "sp-runtime-interface-proc-macro", + "sp-std", + "sp-wasm-interface", + "static_assertions", ] [[package]] name = "sp-runtime-interface-proc-macro" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "Inflector", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", ] [[package]] name = "sp-state-machine" -version = "0.8.0" +version = "0.8.0-alpha.3" dependencies = [ - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-core 2.0.0", - "sp-externalities 0.8.0", - "sp-panic-handler 2.0.0", - "sp-trie 2.0.0", - "trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db", + "log", + "num-traits", + "parity-scale-codec", + "parking_lot 0.10.0", + "rand 0.7.3", + "smallvec 1.2.0", + "sp-core", + "sp-externalities", + "sp-panic-handler", + "sp-trie", + "trie-db", + "trie-root", ] [[package]] name = "sp-state-machine-fuzz" version = "0.0.1" dependencies = [ - "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", - "sp-state-machine 0.8.0", + "libfuzzer-sys", + "sp-state-machine", ] [[package]] name = "sp-std" -version = "2.0.0" +version = "2.0.0-alpha.3" [[package]] name = "sp-storage" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-debug-derive 2.0.0", - "sp-std 2.0.0", + "impl-serde 0.2.3", + "serde", + "sp-debug-derive", + "sp-std", ] [[package]] name = "sp-trie" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "memory-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "sp-core 2.0.0", - "sp-std 2.0.0", - "trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db", + "memory-db", + "parity-scale-codec", + "sp-core", + "sp-std", + "trie-db", + "trie-root", ] [[package]] name = "sp-wasm-interface" -version = "2.0.0" +version = "2.0.0-alpha.3" dependencies = [ - "impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "impl-trait-for-tuples", + "parity-scale-codec", + "sp-std", + "wasmi", ] [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "substrate-bip39" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" dependencies = [ - "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hmac", + "pbkdf2", + "schnorrkel", + "sha2", ] [[package]] name = "subtle" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" [[package]] name = "subtle" -version = "2.2.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" [[package]] name = "syn" -version = "1.0.8" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "unicode-xid", ] [[package]] name = "synstructure" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" dependencies = [ - "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro2", + "quote", + "syn", + "unicode-xid", ] [[package]] name = "thread_local" -version = "0.3.6" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" dependencies = [ - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static", ] [[package]] name = "tiny-bip39" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" dependencies = [ - "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "once_cell 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", - "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "failure", + "hmac", + "once_cell", + "pbkdf2", + "rand 0.7.3", + "rustc-hash", + "sha2", + "unicode-normalization", ] [[package]] name = "tiny-keccak" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" dependencies = [ - "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "crunchy", ] [[package]] name = "toml" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" dependencies = [ - "serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", ] [[package]] name = "trie-db" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" dependencies = [ - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db", + "hashbrown", + "log", + "rustc-hex", + "smallvec 1.2.0", ] [[package]] name = "trie-root" -version = "0.15.2" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" dependencies = [ - "hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db", ] [[package]] name = "twox-hash" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" dependencies = [ - "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3", ] [[package]] name = "typenum" version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" [[package]] name = "uint" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" +dependencies = [ + "byteorder", + "crunchy", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ - "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.2.0", ] [[package]] name = "unicode-xid" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" [[package]] name = "wasi" -version = "0.7.0" +version = "0.9.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasmi" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" dependencies = [ - "libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)", - "memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmi-validation 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "memory_units", + "num-rational", + "num-traits", + "parity-wasm", + "wasmi-validation", ] [[package]] name = "wasmi-validation" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" dependencies = [ - "parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-wasm", ] [[package]] name = "winapi" version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" dependencies = [ - "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", ] [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "zeroize" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" [[package]] name = "zeroize" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" +dependencies = [ + "zeroize_derive", +] -[metadata] -"checksum Inflector 0.11.4 (registry+https://github.com/rust-lang/crates.io-index)" = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" -"checksum ahash 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" -"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" -"checksum arbitrary 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6c7d1523aa3a127adf8b27af2404c03c12825b4c4d0698f01648d63fa9df62ee" -"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" -"checksum arrayvec 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" -"checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" -"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" -"checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea" -"checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491" -"checksum base58 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" -"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" -"checksum bitvec 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" -"checksum blake2-rfc 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" -"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" -"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" -"checksum byte-slice-cast 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f6209f3b2c1edea170002e016d5ead6903d3bb0a846477f53bbeb614967a52a9" -"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" -"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" -"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb" -"checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8" -"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" -"checksum clear_on_drop 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" -"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -"checksum const-random 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7b641a8c9867e341f3295564203b1c250eb8ce6cb6126e007941f78c4d2ed7fe" -"checksum const-random-macro 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c750ec12b83377637110d5a57f5ae08e895b06c4b16e2bdbf1a94ef717428c59" -"checksum constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "995a44c877f9212528ccc74b21a232f66ad69001e40ede5bcee2ac9ef2657120" -"checksum crunchy 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" -"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" -"checksum curve25519-dalek 1.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" -"checksum curve25519-dalek 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" -"checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" -"checksum ed25519-dalek 1.0.0-pre.3 (registry+https://github.com/rust-lang/crates.io-index)" = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" -"checksum environmental 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" -"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" -"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" -"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" -"checksum fixed-hash 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "72fe7539e2c5692c6989f2f9c0457e42f1e5768f96b85c87d273574670ae459f" -"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" -"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" -"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407" -"checksum hash-db 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" -"checksum hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" -"checksum hashbrown 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "3bae29b6653b3412c2e71e9d486db9f9df5d701941d86683005efb9f2d28e3da" -"checksum hashbrown 0.6.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" -"checksum hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" -"checksum hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" -"checksum hmac-drbg 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" -"checksum impl-codec 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" -"checksum impl-serde 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "58e3cae7e99c7ff5a995da2cf78dd0a5383740eda71d98cf7b1910c301ac69b8" -"checksum impl-trait-for-tuples 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" -"checksum keccak 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" -"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum libc 0.2.65 (registry+https://github.com/rust-lang/crates.io-index)" = "1a31a0627fdf1f6a39ec0dd577e101440b7db22672c0901fe00a9a6fbb5c24e8" -"checksum libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)" = "" -"checksum libsecp256k1 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2bd9a7c16c9487e710536b699c962f022266347c94201174aa0a7eb0546051aa" -"checksum lock_api 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" -"checksum lock_api 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8912e782533a93a167888781b836336a6ca5da6175c05944c86cf28c31104dc" -"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" -"checksum malloc_size_of_derive 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e37c5d4cd9473c5f4c9c111f033f15d4df9bd378fdf615944e360a4f55a05f0b" -"checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" -"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" -"checksum memory-db 0.18.1 (registry+https://github.com/rust-lang/crates.io-index)" = "881736a0f68a6fae1b596bb066c5bd16d7b3ed645a4dd8ffaefd02f585abaf71" -"checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" -"checksum merlin 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "de2d16d3b15fec5943d1144f861f61f279d165fdd60998ca262913b9bf1c8adb" -"checksum nodrop 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" -"checksum num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f9c3f34cdd24f334cb265d9bf8bfa8a241920d026916785747a92f0e55541a1a" -"checksum num-integer 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "b85e541ef8255f6cf42bbfe4ef361305c6c135d10919ecc26126c4e5ae94bc09" -"checksum num-rational 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2885278d5fe2adc2f75ced642d52d879bffaceb5a2e0b1d4309ffdfb239b454" -"checksum num-traits 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "443c53b3c3531dfcbfa499d8893944db78474ad7a1d87fa2d94d1a2231693ac6" -"checksum once_cell 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "532c29a261168a45ce28948f9537ddd7a5dd272cc513b3017b1e82a88f962c37" -"checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" -"checksum parity-scale-codec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f9f9d99dae413590a5f37e43cd99b94d4e62a244160562899126913ea7108673" -"checksum parity-scale-codec-derive 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "492ac3aa93d6caa5d20e4e3e0b75d08e2dcd9dd8a50d19529548b6fe11b3f295" -"checksum parity-util-mem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8174d85e62c4d615fddd1ef67966bdc5757528891d0742f15b131ad04667b3f9" -"checksum parity-wasm 0.41.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" -"checksum parking_lot 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" -"checksum parking_lot 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" -"checksum parking_lot_core 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" -"checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" -"checksum pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" -"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" -"checksum primitive-types 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a0253db64c26d8b4e7896dd2063b516d2a1b9e0a5da26b5b78335f236d1e9522" -"checksum proc-macro-crate 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" -"checksum proc-macro-hack 0.5.11 (registry+https://github.com/rust-lang/crates.io-index)" = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" -"checksum proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9e470a8dc4aeae2dee2f335e8f533e2d4b347e1434e5671afc49b054592f27" -"checksum quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "053a8c8bcc71fcce321828dc897a98ab9760bef03a4fc36693c231e5b3216cfe" -"checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -"checksum rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae1b169243eaf61759b8475a998f0a385e42042370f3a7dbaf35246eacc8412" -"checksum rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" -"checksum rand_chacha 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "03a2a90da8c7523f554344f921aa97283eadf6ac484a6d2a7d0212fa7f8d6853" -"checksum rand_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -"checksum rand_core 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" -"checksum rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -"checksum rand_hc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" -"checksum rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -"checksum rand_isaac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" -"checksum rand_jitter 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -"checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -"checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -"checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" -"checksum regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dc220bd33bdce8f093101afe22a037b8eb0e5af33592e6a9caafff0d4cb81cbd" -"checksum regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "11a7e20d1cce64ef2fed88b66d347f88bd9babb82845b2b858f3edbf59a4f716" -"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" -"checksum rustc-hex 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" -"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -"checksum schnorrkel 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)" = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" -"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" -"checksum scopeguard 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b42e15e59b18a828bbf5c58ea01debb36b9b096346de35d941dcb89009f24a0d" -"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" -"checksum serde 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4b39bd9b0b087684013a792c59e3e07a46a01d2322518d8a1104641a0b1be0" -"checksum serde_derive 1.0.102 (registry+https://github.com/rust-lang/crates.io-index)" = "ca13fc1a832f793322228923fbb3aba9f3f44444898f835d31ad1b74fa0a2bf8" -"checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" -"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" -"checksum smallvec 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "44e59e0c9fa00817912ae6e4e6e3c4fe04455e75699d06eedc7d85917ed8e8f4" -"checksum static_assertions 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -"checksum substrate-bip39 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" -"checksum subtle 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" -"checksum subtle 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab3af2eb31c42e8f0ccf43548232556c42737e01a96db6e1777b0be108e79799" -"checksum syn 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "661641ea2aa15845cddeb97dad000d22070bb5c1fb456b96c1cba883ec691e92" -"checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" -"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" -"checksum tiny-bip39 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c1c5676413eaeb1ea35300a0224416f57abc3bd251657e0fafc12c47ff98c060" -"checksum tiny-keccak 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" -"checksum toml 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "01d1404644c8b12b16bfcffa4322403a91a451584daaaa7c28d3152e6cbc98cf" -"checksum trie-db 0.19.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a0d747ae5b6f078df7e46477fcc7df66df9eb4f27a031cf4a7c890a8dd03d8e6" -"checksum trie-root 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0b779f7c1c8fe9276365d9d5be5c4b5adeacf545117bb3f64c974305789c5c0b" -"checksum twox-hash 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" -"checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" -"checksum uint 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" -"checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" -"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d" -"checksum wasmi 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" -"checksum wasmi-validation 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" -"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" -"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" -"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -"checksum zeroize 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" -"checksum zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" +[[package]] +name = "zeroize_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] From 45d5c7cf7a2f575f6a9cf286db4e7cca918b3075 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 14 Apr 2020 18:15:20 +0200 Subject: [PATCH 129/129] fix bench --- primitives/state-machine/src/overlayed_changes.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index df135c5d36e69..b3ef8dd2c5dbc 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -504,7 +504,7 @@ impl OverlayedChanges { /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_storage(&mut self, key: StorageKey, value: Option) { + pub fn set_storage(&mut self, key: StorageKey, value: Option) { let size_write = value.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_write_overlay(size_write); self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref());