diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c141c51d1296..518b4d9d5557b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,9 +168,12 @@ cargo-check-benches: <<: *docker-env script: - BUILD_DUMMY_WASM_BINARY=1 time cargo +nightly check --benches --all + - cd ./primitives/state-machine/fuzz + - time cargo check - sccache -s + cargo-check-subkey: stage: test <<: *docker-env diff --git a/Cargo.lock b/Cargo.lock index a696744bf2d24..3ed0eb1b27480 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -742,7 +742,7 @@ dependencies = [ "gimli", "log", "serde", - "smallvec 1.2.0", + "smallvec 1.3.0", "target-lexicon", "thiserror", ] @@ -780,7 +780,7 @@ checksum = "518344698fa6c976d853319218415fdfb4f1bc6b42d0b2e2df652e55dff1f778" dependencies = [ "cranelift-codegen", "log", - "smallvec 1.2.0", + "smallvec 1.3.0", "target-lexicon", ] @@ -1879,7 +1879,7 @@ dependencies = [ "byteorder 1.3.4", "fallible-iterator", "indexmap", - "smallvec 1.2.0", + "smallvec 1.3.0", "stable_deref_trait", ] @@ -2499,7 +2499,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cad096c6849b2ef027fabe35c4aed356d0e3d3f586d0a8361e5e17f1e50a7ce5" dependencies = [ "parity-util-mem", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -2529,7 +2529,7 @@ dependencies = [ "parking_lot 0.10.0", "regex", "rocksdb", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -2635,7 +2635,7 @@ dependencies = [ "parity-multiaddr 0.8.0", "parking_lot 0.10.0", "pin-project", - "smallvec 1.2.0", + "smallvec 1.3.0", "wasm-timer", ] @@ -2666,7 +2666,7 @@ dependencies = [ "ring", "rw-stream-sink", "sha2", - "smallvec 1.2.0", + "smallvec 1.3.0", "thiserror", "unsigned-varint", "void", @@ -2719,7 +2719,7 @@ dependencies = [ "prost", "prost-build", "rand 0.7.3", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -2742,7 +2742,7 @@ dependencies = [ "prost-build", "rand 0.7.3", "sha2", - "smallvec 1.2.0", + "smallvec 1.3.0", "unsigned-varint", "wasm-timer", ] @@ -2759,7 +2759,7 @@ dependencies = [ "log", "prost", "prost-build", - "smallvec 1.2.0", + "smallvec 1.3.0", "wasm-timer", ] @@ -2783,7 +2783,7 @@ dependencies = [ "prost-build", "rand 0.7.3", "sha2", - "smallvec 1.2.0", + "smallvec 1.3.0", "uint", "unsigned-varint", "void", @@ -2807,7 +2807,7 @@ dependencies = [ "log", "net2", "rand 0.7.3", - "smallvec 1.2.0", + "smallvec 1.3.0", "void", "wasm-timer", ] @@ -2936,7 +2936,7 @@ dependencies = [ "libp2p-core", "log", "rand 0.7.3", - "smallvec 1.2.0", + "smallvec 1.3.0", "void", "wasm-timer", ] @@ -3303,7 +3303,7 @@ dependencies = [ "futures 0.3.4", "log", "pin-project", - "smallvec 1.2.0", + "smallvec 1.3.0", "unsigned-varint", ] @@ -4782,7 +4782,7 @@ dependencies = [ "parity-util-mem-derive", "parking_lot 0.10.0", "primitive-types", - "smallvec 1.2.0", + "smallvec 1.3.0", "winapi 0.3.8", ] @@ -4858,7 +4858,7 @@ dependencies = [ "cloudabi", "libc", "redox_syscall", - "smallvec 1.2.0", + "smallvec 1.3.0", "winapi 0.3.8", ] @@ -5300,7 +5300,7 @@ dependencies = [ "rand_isaac", "rand_jitter", "rand_os", - "rand_pcg", + "rand_pcg 0.1.2", "rand_xorshift", "winapi 0.3.8", ] @@ -5316,6 +5316,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc 0.2.0", + "rand_pcg 0.2.1", ] [[package]] @@ -5425,6 +5426,15 @@ dependencies = [ "rand_core 0.4.2", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_xorshift" version = "0.1.1" @@ -7056,9 +7066,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +checksum = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" [[package]] name = "snow" @@ -7093,7 +7103,7 @@ dependencies = [ "log", "rand 0.7.3", "sha1", - "smallvec 1.2.0", + "smallvec 1.3.0", "static_assertions", "thiserror", ] @@ -7632,6 +7642,7 @@ dependencies = [ name = "sp-state-machine" version = "0.8.0-alpha.5" dependencies = [ + "criterion 0.2.11", "hash-db", "hex-literal", "log", @@ -7639,6 +7650,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.10.0", "rand 0.7.3", + "smallvec 1.3.0", "sp-core", "sp-externalities", "sp-panic-handler", @@ -8748,7 +8760,7 @@ dependencies = [ "hashbrown", "log", "rustc-hex", - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] @@ -8852,7 +8864,7 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" dependencies = [ - "smallvec 1.2.0", + "smallvec 1.3.0", ] [[package]] diff --git a/client/src/client.rs b/client/src/client.rs index 4711ad8b3f1a0..ea23f45737671 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -953,7 +953,7 @@ impl Client where .trigger( ¬ify_import.hash, storage_changes.0.into_iter(), - storage_changes.1.into_iter().map(|(sk, v)| (sk, v.into_iter())), + storage_changes.1.into_iter().map(|(storage_key, v)| (storage_key, v.into_iter())), ); } diff --git a/frame/support/src/storage/mod.rs b/frame/support/src/storage/mod.rs index efec36b540abc..8951345bed2cb 100644 --- a/frame/support/src/storage/mod.rs +++ b/frame/support/src/storage/mod.rs @@ -27,6 +27,21 @@ pub mod child; pub mod generator; pub mod migration; +/// Execute under a new transactional layer. +/// +/// If the result of execution is an error, +/// the transactional layer get reverted; otherwise +/// it is committed. +pub fn with_transaction(f: impl FnOnce() -> Result) -> Result { + sp_io::storage_transaction::start_transaction(); + let result = f(); + match result { + Ok(_) => sp_io::storage_transaction::commit_transaction(), + Err(_) => sp_io::storage_transaction::discard_transaction(), + } + result +} + /// A trait for working with macro-generated storage values under the substrate storage API. /// /// Details on implementation can be found at diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index a907ac8095720..343300b8046f8 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -105,7 +105,6 @@ fn calling_with_both_strategy_and_fail_on_native_should_work() { assert_eq!(runtime_api.fail_on_native(&block_id).unwrap(), 1); } - #[test] fn calling_with_native_else_wasm_and_fail_on_wasm_should_work() { let client = TestClientBuilder::new().set_execution_strategy(ExecutionStrategy::NativeElseWasm).build(); diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 6a7f94394710f..928e69c2e9a48 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -37,12 +37,12 @@ mod scope_limited; /// Provides access to the storage and to other registered extensions. pub trait Externalities: ExtensionStore { /// Read runtime storage. - fn storage(&self, key: &[u8]) -> Option>; + fn storage(&mut self, key: &[u8]) -> Option>; /// Get storage value hash. /// /// This may be optimized for large values. - fn storage_hash(&self, key: &[u8]) -> Option>; + fn storage_hash(&mut self, key: &[u8]) -> Option>; /// Get child storage value hash. /// @@ -50,7 +50,7 @@ pub trait Externalities: ExtensionStore { /// /// Returns an `Option` that holds the SCALE encoded hash. fn child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -60,7 +60,7 @@ pub trait Externalities: ExtensionStore { /// /// Returns an `Option` that holds the SCALE encoded hash. fn child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -98,13 +98,13 @@ pub trait Externalities: ExtensionStore { } /// Whether a storage entry exists. - fn exists_storage(&self, key: &[u8]) -> bool { + fn exists_storage(&mut self, key: &[u8]) -> bool { self.storage(key).is_some() } /// Whether a child storage entry exists. fn exists_child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -113,11 +113,11 @@ pub trait Externalities: ExtensionStore { } /// Returns the key immediately following the given key, if it exists. - fn next_storage_key(&self, key: &[u8]) -> Option>; + fn next_storage_key(&mut self, key: &[u8]) -> Option>; /// Returns the key immediately following the given key, if it exists, in child storage. fn next_child_storage_key( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -177,6 +177,19 @@ pub trait Externalities: ExtensionStore { /// The returned hash is defined by the `Block` and is SCALE encoded. fn storage_changes_root(&mut self, parent: &[u8]) -> Result>, ()>; + /// Start a new transactional layer. + fn storage_start_transaction(&mut self); + + /// Discard current transactional layer, pending changes of this transaction layer are + /// dropped. + /// The transactional layer is closed, a new one is started if this was the last layer. + fn storage_discard_transaction(&mut self); + + /// Commit current transactional layer. + /// The changes stay attached to parent transaction layer and could still be discarded. + /// The transactional layer is closed, a new one is started if this was the last layer. + fn storage_commit_transaction(&mut self); + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! /// Benchmarking related functionality and shouldn't be used anywhere else! /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index bc49df159eb6f..d11da8dfe2609 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -85,7 +85,7 @@ fn child_storage_key_or_panic(storage_key: &[u8]) -> ChildStorageKey { #[runtime_interface] pub trait Storage { /// Returns the data for `key` in the storage or `None` if the key can not be found. - fn get(&self, key: &[u8]) -> Option> { + fn get(&mut self, key: &[u8]) -> Option> { self.storage(key).map(|s| s.to_vec()) } @@ -101,7 +101,7 @@ pub trait Storage { /// This function specifically returns the data for `key` in the child storage or `None` /// if the key can not be found. fn child_get( - &self, + &mut self, child_storage_key: &[u8], child_definition: &[u8], child_type: u32, @@ -118,7 +118,7 @@ pub trait Storage { /// doesn't exist at all. /// If `value_out` length is smaller than the returned length, only `value_out` length bytes /// are copied into `value_out`. - fn read(&self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { + fn read(&mut self, key: &[u8], value_out: &mut [u8], value_offset: u32) -> Option { self.storage(key).map(|value| { let value_offset = value_offset as usize; let data = &value[value_offset.min(value.len())..]; @@ -136,7 +136,7 @@ pub trait Storage { /// /// See `child_get` for common child api parameters. fn child_read( - &self, + &mut self, child_storage_key: &[u8], child_definition: &[u8], child_type: u32, @@ -216,7 +216,7 @@ pub trait Storage { } /// Check whether the given `key` exists in storage. - fn exists(&self, key: &[u8]) -> bool { + fn exists(&mut self, key: &[u8]) -> bool { self.exists_storage(key) } @@ -224,7 +224,7 @@ pub trait Storage { /// /// See `child_get` for common child api parameters. fn child_exists( - &self, + &mut self, child_storage_key: &[u8], child_definition: &[u8], child_type: u32, @@ -313,6 +313,25 @@ pub trait Storage { } } +/// Interface for managing transaction within the runtime. +#[runtime_interface] +pub trait StorageTransaction { + /// Start a new transaction. + fn start_transaction(&mut self) { + self.storage_start_transaction(); + } + + /// Discard a transactional layer. + fn discard_transaction(&mut self) { + self.storage_discard_transaction(); + } + + /// Commit a transactional layer. + fn commit_transaction(&mut self) { + self.storage_commit_transaction(); + } +} + /// Interface that provides trie related functionality. #[runtime_interface] pub trait Trie { @@ -933,6 +952,7 @@ pub type TestExternalities = sp_state_machine::TestExternalities. + +use criterion::{Criterion, black_box, Bencher}; +use criterion::{criterion_group, criterion_main}; +use sp_state_machine::OverlayedChanges; + +const CONTENT_KEY_SIZE: usize = 32; + +fn get_content(seed: u64, len: usize) -> Vec { + use rand::SeedableRng; + use rand::RngCore; + let mut rng = rand::rngs::SmallRng::seed_from_u64(seed); + let mut data = vec![0u8; len]; + rng.fill_bytes(&mut data[..]); + data +} + +fn key_val( + mut inp: &[u8], + len_key: usize, + len_val: usize, +) -> Vec<(Vec, Vec)> { + let mut result = Vec::new(); + loop { + let key = if inp.len() >= len_key { + inp[..len_key].to_vec() + } else { break }; + inp = &inp[len_key..]; + let val = if inp.len() >= len_val { + inp[..len_val].to_vec() + } else { break }; + inp = &inp[len_val..]; + result.push((key, val)); + } + result +} + +fn commit_drop_commit(b: &mut Bencher, input: &Vec) { + let key_vals = key_val(&input[..], 32, 32); + b.iter(move || { + let mut overlayed = OverlayedChanges::default(); + for i in key_vals.iter() { + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.commit_prospective(); + for i in key_vals.iter() { + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.discard_prospective(); + for i in key_vals.iter() { + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.commit_prospective(); + }); +} + +fn commit_drop_commit_and_get(b: &mut Bencher, input: &Vec) { + let key_vals = key_val(&input[..], 32, 32); + b.iter(move || { + let mut overlayed = OverlayedChanges::default(); + for i in key_vals.iter() { + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); + } + for i in key_vals.iter() { + black_box(overlayed.storage(&i.0)); + } + overlayed.commit_prospective(); + for i in key_vals.iter() { + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); + } + for i in key_vals.iter() { + black_box(overlayed.storage(&i.0)); + } + overlayed.discard_prospective(); + for i in key_vals.iter() { + black_box(overlayed.storage(&i.0)); + } + for i in key_vals.iter() { + overlayed.set_storage(i.0.clone(), Some(i.1.clone())); + } + overlayed.commit_prospective(); + }); +} + +fn bench_overlay_commit_drop_commit(c: &mut Criterion) { + let inp = get_content(12, CONTENT_KEY_SIZE * 100); + let inps = vec![inp]; + c.bench_function_over_inputs("commit_drop_commit", commit_drop_commit, inps); +} + +fn bench_overlay_commit_drop_commit_get(c: &mut Criterion) { + let inp = get_content(12, CONTENT_KEY_SIZE * 100); + let inps = vec![inp]; + c.bench_function_over_inputs("commit_drop_commit_get", commit_drop_commit_and_get, inps); +} + +criterion_group!(benches, + bench_overlay_commit_drop_commit, + bench_overlay_commit_drop_commit_get, +); + +criterion_main!(benches); diff --git a/primitives/state-machine/fuzz/Cargo.lock b/primitives/state-machine/fuzz/Cargo.lock new file mode 100644 index 0000000000000..c7ff1e839534a --- /dev/null +++ b/primitives/state-machine/fuzz/Cargo.lock @@ -0,0 +1,1460 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + +[[package]] +name = "ahash" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f33b5018f120946c1dcf279194f238a9f146725593ead1c08fa47ff22b0b5d3" +dependencies = [ + "const-random", +] + +[[package]] +name = "aho-corasick" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" +dependencies = [ + "memchr", +] + +[[package]] +name = "arbitrary" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cf76cb6e2222ed0ea86b2b0ee2f71c96ec6edd5af42e84d59160e91b836ec4" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9" +dependencies = [ + "nodrop", +] + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "backtrace" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad235dabf00f36301792cfe82499880ba54c6486be094d1047b02bacb67c14e8" +dependencies = [ + "backtrace-sys", + "cfg-if", + "libc", + "rustc-demangle", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17b52e737c40a7d75abca20b29a19a0eb7ba9fc72c5a72dd282a0a3c2c0dc35" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "base58" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5024ee8015f02155eee35c711107ddd9a9bf3cb689cf2a9089c97e79b6e1ae83" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitvec" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993f74b4c99c1908d156b8d2e0fb6277736b0ecbd833982fd1241d39b2766a6" + +[[package]] +name = "blake2-rfc" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d530bdd2d52966a6d03b7a964add7ae1a288d25214066fd4b600f0f796400" +dependencies = [ + "arrayvec 0.4.12", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "byte-slice-cast" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0a5e3906bcbf133e33c1d4d95afc664ad37fbdb9f6568d8043e7ea8c27d93d3" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "clear_on_drop" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97276801e127ffb46b66ce23f35cc96bd454fa311294bced4bbace7baa8b1d17" +dependencies = [ + "cc", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "const-random" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f1af9ac737b2dd2d577701e59fd09ba34822f6f2ebdb30a7647405d9e55e16a" +dependencies = [ + "const-random-macro", + "proc-macro-hack", +] + +[[package]] +name = "const-random-macro" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25e4c606eb459dd29f7c57b2e0879f2b6f14ee130918c2b78ccb58a9624e6c7a" +dependencies = [ + "getrandom", + "proc-macro-hack", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +dependencies = [ + "generic-array", + "subtle 1.0.0", +] + +[[package]] +name = "curve25519-dalek" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7dcd30ba50cdf88b55b033456138b7c0ac4afdc436d82e1b79f370f24cc66d" +dependencies = [ + "byteorder", + "clear_on_drop", + "digest", + "rand_core 0.3.1", + "subtle 2.2.2", +] + +[[package]] +name = "curve25519-dalek" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26778518a7f6cffa1d25a44b602b62b979bd88adb9e99ffec546998cf3404839" +dependencies = [ + "byteorder", + "digest", + "rand_core 0.5.1", + "subtle 2.2.2", + "zeroize 1.1.0", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.0-pre.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978710b352437433c97b2bff193f2fb1dfd58a093f863dd95e225a19baa599a2" +dependencies = [ + "clear_on_drop", + "curve25519-dalek 2.0.0", + "rand 0.7.3", + "sha2", +] + +[[package]] +name = "environmental" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516aa8d7a71cb00a1c4146f0798549b93d083d4f189b3ced8f3de6b8f11ee6c4" + +[[package]] +name = "failure" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8529c2421efa3066a5cbd8063d2244603824daccb6936b079010bb2aa89464b" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a733c8287d6213886dd487564ff5c8f6aae10278b3588ed177f9d18f8d231" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fixed-hash" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3367952ceb191f4ab95dd5685dc163ac539e36202f9fcfd0cb22f9f9c542fefc" +dependencies = [ + "byteorder", + "libc", + "rand 0.7.3", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hash-db" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" + +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c171d55b98633f4ed3860808f004099b36c1cc29c42cfc53aa8591b21efcf2" +dependencies = [ + "crunchy", +] + +[[package]] +name = "hashbrown" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6073d0ca812575946eb5f35ff68dbe519907b25c42530389ff946dc84c6ead" +dependencies = [ + "ahash", + "autocfg 0.1.7", +] + +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + +[[package]] +name = "hmac" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "hmac-drbg" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6e570451493f10f6581b48cdd530413b63ea9e780f544bfd3bdcaa0d89d1a7b" +dependencies = [ + "digest", + "generic-array", + "hmac", +] + +[[package]] +name = "impl-codec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1be51a921b067b0eaca2fad532d9400041561aa922221cc65f95a85641c6bf53" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-serde" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58e3cae7e99c7ff5a995da2cf78dd0a5383740eda71d98cf7b1910c301ac69b8" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-serde" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bbe9ea9b182f0fb1cabbd61f4ff9b7b7b9197955e95a7e4c27de5055eb29ff8" +dependencies = [ + "serde", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef5550a42e3740a0e71f909d4c861056a284060af885ae7aa6242820f920d9d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "keccak" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c21572b4949434e4fc1e1978b99c5f77064153c59d998bf13ecd96fb5ecba7" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb147597cdf94ed43ab7a9038716637d2d1bf2bc571da995d0028dec06bd3018" + +[[package]] +name = "libfuzzer-sys" +version = "0.1.0" +source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#35ce7d7177c254b9c798aec171dfe76877d1a20f" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libsecp256k1" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc1e2c808481a63dc6da2074752fdd4336a3c8fcc68b83db6f1fd5224ae7962" +dependencies = [ + "arrayref", + "crunchy", + "digest", + "hmac-drbg", + "rand 0.7.3", + "sha2", + "subtle 2.2.2", + "typenum", +] + +[[package]] +name = "lock_api" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "memory-db" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "198831fe8722331a395bc199a5d08efbc197497ef354cb4c77b969c02ffc0fc4" +dependencies = [ + "ahash", + "hash-db", + "hashbrown", + "parity-util-mem", +] + +[[package]] +name = "memory_units" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" + +[[package]] +name = "merlin" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0942b357c1b4d0dc43ba724674ec89c3218e6ca2b3e8269e7cb53bcecd2f6e" +dependencies = [ + "byteorder", + "keccak", + "rand_core 0.4.2", + "zeroize 1.1.0", +] + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.0.0", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6ea62e9d81a77cd3ee9a2a5b9b609447857f3d358704331e4ef39eb247fcba" +dependencies = [ + "autocfg 1.0.0", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da4dc79f9e6c81bef96148c8f6b8e72ad4541caa4a24373e900a36da07de03a3" +dependencies = [ + "autocfg 1.0.0", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" +dependencies = [ + "autocfg 1.0.0", +] + +[[package]] +name = "once_cell" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c601810575c99596d4afc46f78a678c80105117c379eb3650cf99b8a21ce5b" +dependencies = [ + "parking_lot 0.9.0", +] + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "parity-scale-codec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f509c5e67ca0605ee17dcd3f91ef41cadd685c75a298fb6261b781a5acb3f910" +dependencies = [ + "arrayvec 0.5.1", + "bitvec", + "byte-slice-cast", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a0ec292e92e8ec7c58e576adacc1e3f399c597c8f263c42f18420abe58e7245" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "parity-util-mem" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef1476e40bf8f5c6776e9600983435821ca86eb9819d74a6207cca69d091406a" +dependencies = [ + "cfg-if", + "impl-trait-for-tuples", + "parity-util-mem-derive", + "parking_lot 0.10.0", + "primitive-types", + "winapi", +] + +[[package]] +name = "parity-util-mem-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f557c32c6d268a07c921471619c0295f5efad3a0e76d4f97a05c091a51d110b2" +dependencies = [ + "proc-macro2", + "syn", + "synstructure", +] + +[[package]] +name = "parity-wasm" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc878dac00da22f8f61e7af3157988424567ab01d9920b962ef7dcbd7cd865" + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core 0.6.2", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" +dependencies = [ + "lock_api", + "parking_lot_core 0.7.0", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec 0.6.13", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "smallvec 1.2.0", + "winapi", +] + +[[package]] +name = "pbkdf2" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" +dependencies = [ + "byteorder", + "crypto-mac", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" + +[[package]] +name = "primitive-types" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4336f4f5d5524fa60bcbd6fe626f9223d8142a50e7053e979acdf0da41ab975" +dependencies = [ + "fixed-hash", + "impl-codec", + "impl-serde 0.3.0", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10d4b51f154c8a7fb96fd6dad097cb74b863943ec010ac94b9fd1be8861fe1e" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd45702f76d6d3c75a80564378ae228a85f0b59d2f3ed43c91b4a69eb2ebfc5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c09721c6781493a2a492a96b5a5bf19b65917fe6728884e7c44dd0c60ca3435" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "regex" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322cf97724bea3ee221b78fe25ac9c46114ebb51747ad5babd51a2fc6a8235a8" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1132f845907680735a84409c3bebc64d1364a5683ffbce899550cd09d5eaefc1" + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "schnorrkel" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eacd8381b3c37840c9c9f40472af529e49975bdcbc24f83c31059fd6539023d3" +dependencies = [ + "curve25519-dalek 1.2.3", + "failure", + "merlin", + "rand 0.6.5", + "rand_core 0.4.2", + "rand_os", + "sha2", + "subtle 2.2.2", + "zeroize 0.9.3", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sha2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" + +[[package]] +name = "sp-core" +version = "2.0.0-alpha.3" +dependencies = [ + "base58", + "blake2-rfc", + "byteorder", + "ed25519-dalek", + "hash-db", + "hash256-std-hasher", + "hex", + "impl-serde 0.3.0", + "lazy_static", + "libsecp256k1", + "log", + "num-traits", + "parity-scale-codec", + "parity-util-mem", + "parking_lot 0.10.0", + "primitive-types", + "rand 0.7.3", + "regex", + "rustc-hex", + "schnorrkel", + "serde", + "sha2", + "sp-debug-derive", + "sp-externalities", + "sp-runtime-interface", + "sp-std", + "sp-storage", + "substrate-bip39", + "tiny-bip39", + "tiny-keccak", + "twox-hash", + "wasmi", + "zeroize 1.1.0", +] + +[[package]] +name = "sp-debug-derive" +version = "2.0.0-alpha.3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-externalities" +version = "0.8.0-alpha.3" +dependencies = [ + "environmental", + "sp-std", + "sp-storage", +] + +[[package]] +name = "sp-panic-handler" +version = "2.0.0-alpha.3" +dependencies = [ + "backtrace", + "log", +] + +[[package]] +name = "sp-runtime-interface" +version = "2.0.0-alpha.3" +dependencies = [ + "parity-scale-codec", + "primitive-types", + "sp-externalities", + "sp-runtime-interface-proc-macro", + "sp-std", + "sp-wasm-interface", + "static_assertions", +] + +[[package]] +name = "sp-runtime-interface-proc-macro" +version = "2.0.0-alpha.3" +dependencies = [ + "Inflector", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sp-state-machine" +version = "0.8.0-alpha.3" +dependencies = [ + "hash-db", + "log", + "num-traits", + "parity-scale-codec", + "parking_lot 0.10.0", + "rand 0.7.3", + "smallvec 1.2.0", + "sp-core", + "sp-externalities", + "sp-panic-handler", + "sp-trie", + "trie-db", + "trie-root", +] + +[[package]] +name = "sp-state-machine-fuzz" +version = "0.0.1" +dependencies = [ + "libfuzzer-sys", + "sp-state-machine", +] + +[[package]] +name = "sp-std" +version = "2.0.0-alpha.3" + +[[package]] +name = "sp-storage" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-serde 0.2.3", + "serde", + "sp-debug-derive", + "sp-std", +] + +[[package]] +name = "sp-trie" +version = "2.0.0-alpha.3" +dependencies = [ + "hash-db", + "memory-db", + "parity-scale-codec", + "sp-core", + "sp-std", + "trie-db", + "trie-root", +] + +[[package]] +name = "sp-wasm-interface" +version = "2.0.0-alpha.3" +dependencies = [ + "impl-trait-for-tuples", + "parity-scale-codec", + "sp-std", + "wasmi", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "substrate-bip39" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be511be555a3633e71739a79e4ddff6a6aaa6579fa6114182a51d72c3eb93c5" +dependencies = [ + "hmac", + "pbkdf2", + "schnorrkel", + "sha2", +] + +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" + +[[package]] +name = "subtle" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c65d530b10ccaeac294f349038a597e435b18fb456aadd0840a623f83b9e941" + +[[package]] +name = "syn" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "123bd9499cfb380418d509322d7a6d52e5315f064fe4b3ad18a53d6b92c07859" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tiny-bip39" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6848cd8f566953ce1e8faeba12ee23cbdbb0437754792cd857d44628b5685e3" +dependencies = [ + "failure", + "hmac", + "once_cell", + "pbkdf2", + "rand 0.7.3", + "rustc-hash", + "sha2", + "unicode-normalization", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2953ca5148619bc99695c1274cb54c5275bbb913c6adad87e72eaf8db9787f69" +dependencies = [ + "crunchy", +] + +[[package]] +name = "toml" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" +dependencies = [ + "serde", +] + +[[package]] +name = "trie-db" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9222c50cc325855621271157c973da27a0dcd26fa06f8edf81020bd2333df0" +dependencies = [ + "hash-db", + "hashbrown", + "log", + "rustc-hex", + "smallvec 1.2.0", +] + +[[package]] +name = "trie-root" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "652931506d2c1244d7217a70b99f56718a7b4161b37f04e7cd868072a99f68cd" +dependencies = [ + "hash-db", +] + +[[package]] +name = "twox-hash" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bfd5b7557925ce778ff9b9ef90e3ade34c524b5ff10e239c69a42d546d2af56" +dependencies = [ + "rand 0.7.3", +] + +[[package]] +name = "typenum" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" + +[[package]] +name = "uint" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75a4cdd7b87b28840dba13c483b9a88ee6bbf16ba5c951ee1ecfcf723078e0d" +dependencies = [ + "byteorder", + "crunchy", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5479532badd04e128284890390c1e876ef7a993d0570b3597ae43dfa1d59afa4" +dependencies = [ + "smallvec 1.2.0", +] + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasmi" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf617d864d25af3587aa745529f7aaa541066c876d57e050c0d0c85c61c92aff" +dependencies = [ + "libc", + "memory_units", + "num-rational", + "num-traits", + "parity-wasm", + "wasmi-validation", +] + +[[package]] +name = "wasmi-validation" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea78c597064ba73596099281e2f4cfc019075122a65cdda3205af94f0b264d93" +dependencies = [ + "parity-wasm", +] + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "zeroize" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45af6a010d13e4cf5b54c94ba5a2b2eba5596b9e46bf5875612d332a1f2b3f86" + +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de251eec69fc7c1bc3923403d18ececb929380e016afe103da75f396704f8ca2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/primitives/state-machine/fuzz/Cargo.toml b/primitives/state-machine/fuzz/Cargo.toml new file mode 100644 index 0000000000000..d8ecf770b59a7 --- /dev/null +++ b/primitives/state-machine/fuzz/Cargo.toml @@ -0,0 +1,29 @@ + +[package] +name = "sp-state-machine-fuzz" +version = "0.0.1" +authors = [] +publish = false +edition = "2018" + +[package.metadata] +cargo-fuzz = true + +[dependencies.sp-state-machine] +path = ".." +features = ["test-helpers"] + +[dependencies.libfuzzer-sys] +git = "https://github.com/rust-fuzz/libfuzzer-sys.git" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "fuzz_transactions_no_compactness" +path = "fuzz_targets/fuzz_transactions.rs" + +[[bin]] +name = "fuzz_transactions_with_compactness" +path = "fuzz_targets/fuzz_transactions_with_compactness.rs" diff --git a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs new file mode 100644 index 0000000000000..f105df16ad28b --- /dev/null +++ b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use sp_state_machine::transaction_layers_fuzz::fuzz_transactions_inner; + +fuzz_target!(|data: &[u8]| { + fuzz_transactions_inner(data, false) +}); diff --git a/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs new file mode 100644 index 0000000000000..585990a0a4bb2 --- /dev/null +++ b/primitives/state-machine/fuzz/fuzz_targets/fuzz_transactions_with_compactness.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use sp_state_machine::transaction_layers_fuzz::fuzz_transactions_inner; + +fuzz_target!(|data: &[u8]| { + fuzz_transactions_inner(data, true) +}); diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 819244050ba4c..c2fb77b6367ec 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -111,16 +111,16 @@ impl From> for BasicExternalities { } impl Externalities for BasicExternalities { - fn storage(&self, key: &[u8]) -> Option { + fn storage(&mut self, key: &[u8]) -> Option { self.inner.top.get(key).cloned() } - fn storage_hash(&self, key: &[u8]) -> Option> { + fn storage_hash(&mut self, key: &[u8]) -> Option> { self.storage(key).map(|v| Blake2Hasher::hash(&v).encode()) } fn child_storage( - &self, + &mut self, storage_key: ChildStorageKey, _child_info: ChildInfo, key: &[u8], @@ -129,7 +129,7 @@ impl Externalities for BasicExternalities { } fn child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -137,13 +137,13 @@ impl Externalities for BasicExternalities { self.child_storage(storage_key, child_info, key).map(|v| Blake2Hasher::hash(&v).encode()) } - fn next_storage_key(&self, key: &[u8]) -> Option { + fn next_storage_key(&mut self, key: &[u8]) -> Option { let range = (Bound::Excluded(key), Bound::Unbounded); self.inner.top.range::<[u8], _>(range).next().map(|(k, _)| k).cloned() } fn next_child_storage_key( - &self, + &mut self, storage_key: ChildStorageKey, _child_info: ChildInfo, key: &[u8], @@ -273,6 +273,18 @@ impl Externalities for BasicExternalities { Ok(None) } + fn storage_start_transaction(&mut self) { + warn!("No support for storage transaction"); + } + + fn storage_discard_transaction(&mut self) { + panic!("No support for storage transaction"); + } + + fn storage_commit_transaction(&mut self) { + // no need to fail in this case + } + fn wipe(&mut self) {} fn commit(&mut self) {} diff --git a/primitives/state-machine/src/changes_trie/build.rs b/primitives/state-machine/src/changes_trie/build.rs index 39ad81ed59a0e..14d0fd60ca27b 100644 --- a/primitives/state-machine/src/changes_trie/build.rs +++ b/primitives/state-machine/src/changes_trie/build.rs @@ -16,7 +16,7 @@ //! Structures and functions required to build changes trie for given block. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::collections::btree_map::Entry; use codec::{Decode, Encode}; use hash_db::Hasher; @@ -105,19 +105,14 @@ fn prepare_extrinsics_input<'a, B, H, Number>( Number: BlockNumber, { - let mut children_keys = BTreeSet::::new(); let mut children_result = BTreeMap::new(); - for (storage_key, _) in changes.prospective.children.iter() - .chain(changes.committed.children.iter()) { - children_keys.insert(storage_key.clone()); - } - for storage_key in children_keys { + for (storage_key, _, _) in changes.changes.children_iter() { let child_index = ChildIndex:: { block: block.clone(), - storage_key: storage_key.clone(), + storage_key: storage_key.to_vec(), }; - let iter = prepare_extrinsics_input_inner(backend, block, changes, Some(storage_key))?; + let iter = prepare_extrinsics_input_inner(backend, block, changes, Some(storage_key.to_vec()))?; children_result.insert(child_index, iter); } @@ -137,35 +132,26 @@ fn prepare_extrinsics_input_inner<'a, B, H, Number>( H: Hasher, Number: BlockNumber, { - let (committed, prospective, child_info) = if let Some(sk) = storage_key.as_ref() { - let child_info = changes.child_info(sk).cloned(); - ( - changes.committed.children.get(sk).map(|c| &c.0), - changes.prospective.children.get(sk).map(|c| &c.0), - child_info, - ) - } else { - (Some(&changes.committed.top), Some(&changes.prospective.top), None) - }; - committed.iter().flat_map(|c| c.iter()) - .chain(prospective.iter().flat_map(|c| c.iter())) - .filter(|( _, v)| v.extrinsics.is_some()) + let (iter, child_info) = changes.changes.iter_overlay(storage_key.as_ref().map(|r| r.as_slice())); + iter.filter(|( _, v)| v.extrinsics.is_some()) .try_fold(BTreeMap::new(), |mut map: BTreeMap<&[u8], (ExtrinsicIndex, Vec)>, (k, v)| { match map.entry(k) { Entry::Vacant(entry) => { // ignore temporary values (values that have null value at the end of operation // AND are not in storage at the beginning of operation if let Some(sk) = storage_key.as_ref() { - if !changes.child_storage(sk, k).map(|v| v.is_some()).unwrap_or_default() { - if let Some(child_info) = child_info.as_ref() { - if !backend.exists_child_storage(sk, child_info.as_ref(), k) + if !changes.child_storage_once(sk, k).map(|v| v.0.is_some()).unwrap_or_default() { + if let Some(child_info) = child_info { + if !backend.exists_child_storage(sk, child_info, k) .map_err(|e| format!("{}", e))? { return Ok(map); } } } } else { - if !changes.storage(k).map(|v| v.is_some()).unwrap_or_default() { + // we use `storage_once` because this is called a limited number of time + // per block processing. + if !changes.storage_once(k).map(|v| v.0.is_some()).unwrap_or_default() { if !backend.exists_storage(k).map_err(|e| format!("{}", e))? { return Ok(map); } @@ -349,6 +335,8 @@ mod test { use crate::changes_trie::{RootsStorage, Configuration, storage::InMemoryStorage}; use crate::changes_trie::build_cache::{IncompleteCacheAction, IncompleteCachedBuildData}; use crate::overlayed_changes::{OverlayedValue, OverlayedChangeSet}; + use crate::transaction_layers::{COMMITTED_LAYER, Layers, LayerEntry}; + use crate::transaction_layers::{States, TransactionState}; use super::*; const CHILD_INFO_1: ChildInfo<'static> = ChildInfo::new_default(b"unique_id_1"); @@ -420,54 +408,67 @@ mod test { ]), ]); let changes = OverlayedChanges { - prospective: OverlayedChangeSet { top: vec![ - (vec![100], OverlayedValue { - value: Some(vec![200]), - extrinsics: Some(vec![0, 2].into_iter().collect()) - }), - (vec![103], OverlayedValue { - value: None, - extrinsics: Some(vec![0, 1].into_iter().collect()) - }), - ].into_iter().collect(), - children: vec![ - (child_trie_key1.clone(), (vec![ - (vec![100], OverlayedValue { - value: Some(vec![200]), - extrinsics: Some(vec![0, 2].into_iter().collect()) - }) - ].into_iter().collect(), CHILD_INFO_1.to_owned())), - (child_trie_key2, (vec![ - (vec![100], OverlayedValue { + changes: OverlayedChangeSet { + states: States::test_vector( + vec![TransactionState::Pending, TransactionState::Pending], + vec![0, 0], + 1, + ), + top: vec![ + (EXTRINSIC_INDEX.to_vec(), Layers::from_iter(vec![ + (OverlayedValue { + value: Some(3u32.encode()), + extrinsics: None, + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![100], Layers::from_iter(vec![ + (OverlayedValue { + value: Some(vec![202]), + extrinsics: Some(vec![3].into_iter().collect()) + }, COMMITTED_LAYER), + (OverlayedValue { value: Some(vec![200]), - extrinsics: Some(vec![0, 2].into_iter().collect()) - }) - ].into_iter().collect(), CHILD_INFO_2.to_owned())), - ].into_iter().collect() - }, - committed: OverlayedChangeSet { top: vec![ - (EXTRINSIC_INDEX.to_vec(), OverlayedValue { - value: Some(3u32.encode()), - extrinsics: None, - }), - (vec![100], OverlayedValue { - value: Some(vec![202]), - extrinsics: Some(vec![3].into_iter().collect()) - }), - (vec![101], OverlayedValue { - value: Some(vec![203]), - extrinsics: Some(vec![1].into_iter().collect()) - }), - ].into_iter().collect(), + extrinsics: Some(vec![3, 0, 2].into_iter().collect()) + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![101], Layers::from_iter(vec![ + (OverlayedValue { + value: Some(vec![203]), + extrinsics: Some(vec![1].into_iter().collect()) + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![103], Layers::from_iter(vec![ + (OverlayedValue { + value: None, + extrinsics: Some(vec![0, 1].into_iter().collect()) + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + ].into_iter().collect(), children: vec![ (child_trie_key1, (vec![ - (vec![100], OverlayedValue { - value: Some(vec![202]), - extrinsics: Some(vec![3].into_iter().collect()) - }) + (vec![100], Layers::from_iter(vec![ + (OverlayedValue { + value: Some(vec![202]), + extrinsics: Some(vec![3].into_iter().collect()) + }, COMMITTED_LAYER), + (OverlayedValue { + value: Some(vec![200]), + extrinsics: Some(vec![3, 0, 2].into_iter().collect()) + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), ].into_iter().collect(), CHILD_INFO_1.to_owned())), + (child_trie_key2, (vec![ + (vec![100], Layers::from_iter(vec![ + (OverlayedValue { + value: Some(vec![200]), + extrinsics: Some(vec![0, 2].into_iter().collect()) + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + ].into_iter().collect(), CHILD_INFO_2.to_owned())), ].into_iter().collect(), }, + operation_from_last_gc: 0, + not_eager_gc: false, collect_extrinsics: true, stats: Default::default(), }; @@ -660,10 +661,12 @@ mod test { let (backend, storage, mut changes, config) = prepare_for_build(zero); // 110: missing from backend, set to None in overlay - changes.prospective.top.insert(vec![110], OverlayedValue { - value: None, - extrinsics: Some(vec![1].into_iter().collect()) - }); + changes.changes.top.insert(vec![110], Layers::from_iter(vec![ + (OverlayedValue { + value: None, + extrinsics: Some(vec![1].into_iter().collect()), + }, COMMITTED_LAYER + 1), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))); let parent = AnchorBlockId { hash: Default::default(), number: zero + 3 }; let changes_trie_nodes = prepare_input( diff --git a/primitives/state-machine/src/changes_trie/mod.rs b/primitives/state-machine/src/changes_trie/mod.rs index d614992df3033..ee11bfcbaad12 100644 --- a/primitives/state-machine/src/changes_trie/mod.rs +++ b/primitives/state-machine/src/changes_trie/mod.rs @@ -256,9 +256,11 @@ pub fn build_changes_trie<'a, B: Backend, H: Hasher, Number: BlockNumber>( // prepare configuration range - we already know zero block. Current block may be the end block if configuration // has been changed in this block - let is_config_changed = match changes.storage(sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG) { - Some(Some(new_config)) => new_config != &state.config.encode()[..], - Some(None) => true, + let is_config_changed = match changes.storage_once( + sp_core::storage::well_known_keys::CHANGES_TRIE_CONFIG + ) { + Some((Some(new_config), _)) => new_config != &state.config.encode()[..], + Some((None, _)) => true, None => false, }; let config_range = ConfigurationRange { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 133af7ccd9b5f..828b4b267e6a1 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -136,8 +136,10 @@ where self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.committed.top.clone().into_iter().map(|(k, v)| (k, v.value))) - .chain(self.overlay.prospective.top.clone().into_iter().map(|(k, v)| (k, v.value))) + .chain( + self.overlay.changes.iter_values(None) + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))) + ) .collect::>() .into_iter() .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) @@ -152,7 +154,7 @@ where B: 'a + Backend, N: crate::changes_trie::BlockNumber, { - fn storage(&self, key: &[u8]) -> Option { + fn storage(&mut self, key: &[u8]) -> Option { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL)); @@ -164,7 +166,7 @@ where result } - fn storage_hash(&self, key: &[u8]) -> Option> { + fn storage_hash(&mut self, key: &[u8]) -> Option> { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = self.overlay .storage(key) @@ -180,7 +182,7 @@ where } fn child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -205,7 +207,7 @@ where } fn child_storage_hash( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -229,7 +231,7 @@ where result.map(|r| r.encode()) } - fn exists_storage(&self, key: &[u8]) -> bool { + fn exists_storage(&mut self, key: &[u8]) -> bool { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = match self.overlay.storage(key) { Some(x) => x.is_some(), @@ -246,7 +248,7 @@ where } fn exists_child_storage( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], @@ -269,48 +271,52 @@ where result } - fn next_storage_key(&self, key: &[u8]) -> Option { - let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); - let next_overlay_key_change = self.overlay.next_storage_key_change(key); + fn next_storage_key(&mut self, key: &[u8]) -> Option { + let next_key = { + let next_backend_key = self.backend.next_storage_key(key).expect(EXT_NOT_ALLOWED_TO_FAIL); + let next_overlay_key_change = self.overlay.next_storage_key_change(key); - match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), - (backend_key, None) => backend_key, - (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { - Some(overlay_key.0.to_vec()) - } else { - self.next_storage_key(&overlay_key.0[..]) - }, - } + match (next_backend_key, next_overlay_key_change) { + (Some(backend_key), Some(overlay_key)) + if &backend_key[..] < overlay_key.0 => return Some(backend_key), + (backend_key, None) => return backend_key, + (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { + return Some(overlay_key.0.to_vec()) + } else { + overlay_key.0.to_vec() + }, + } + }; + self.next_storage_key(&next_key[..]) } fn next_child_storage_key( - &self, + &mut self, storage_key: ChildStorageKey, child_info: ChildInfo, key: &[u8], ) -> Option { - let next_backend_key = self.backend - .next_child_storage_key(storage_key.as_ref(), child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL); - let next_overlay_key_change = self.overlay.next_child_storage_key_change( - storage_key.as_ref(), - key - ); + let next_key = { + let next_backend_key = self.backend + .next_child_storage_key(storage_key.as_ref(), child_info, key) + .expect(EXT_NOT_ALLOWED_TO_FAIL); + let next_overlay_key_change = self.overlay.next_child_storage_key_change( + storage_key.as_ref(), + key + ); - match (next_backend_key, next_overlay_key_change) { - (Some(backend_key), Some(overlay_key)) if &backend_key[..] < overlay_key.0 => Some(backend_key), - (backend_key, None) => backend_key, - (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { - Some(overlay_key.0.to_vec()) - } else { - self.next_child_storage_key( - storage_key, - child_info, - &overlay_key.0[..], - ) - }, - } + match (next_backend_key, next_overlay_key_change) { + (Some(backend_key), Some(overlay_key)) + if &backend_key[..] < overlay_key.0 => return Some(backend_key), + (backend_key, None) => return backend_key, + (_, Some(overlay_key)) => if overlay_key.1.value.is_some() { + return Some(overlay_key.0.to_vec()) + } else { + overlay_key.0.to_vec() + }, + } + }; + self.next_child_storage_key(storage_key, child_info, &next_key[..]) } fn place_storage(&mut self, key: StorageKey, value: Option) { @@ -444,19 +450,15 @@ where } else { let storage_key = storage_key.as_ref(); - if let Some(child_info) = self.overlay.child_info(storage_key).cloned() { - let (root, is_empty, _) = { - let delta = self.overlay.committed.children.get(storage_key) - .into_iter() - .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value))) - .chain( - self.overlay.prospective.children.get(storage_key) - .into_iter() - .flat_map(|(map, _)| map.clone().into_iter().map(|(k, v)| (k, v.value))) - ); + if let Some(child_info) = self.overlay.child_info(storage_key) { + let delta = self.overlay.changes.iter_values(Some(storage_key)) + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); - self.backend.child_storage_root(storage_key, child_info.as_ref(), delta) - }; + let (root, is_empty, _) = self.backend.child_storage_root( + storage_key, + child_info, + delta, + ); let root = root.encode(); // We store update in the overlay in order to be able to use 'self.storage_transaction' @@ -519,6 +521,24 @@ where root.map(|r| r.map(|o| o.encode())) } + fn storage_start_transaction(&mut self) { + let _guard = sp_panic_handler::AbortGuard::force_abort(); + + self.overlay.start_transaction() + } + + fn storage_discard_transaction(&mut self) { + let _guard = sp_panic_handler::AbortGuard::force_abort(); + + self.overlay.discard_transaction() + } + + fn storage_commit_transaction(&mut self) { + let _guard = sp_panic_handler::AbortGuard::force_abort(); + + self.overlay.commit_transaction() + } + fn wipe(&mut self) { self.overlay.discard_prospective(); self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache) @@ -561,9 +581,10 @@ mod tests { changes_trie::{ Configuration as ChangesTrieConfiguration, InMemoryStorage as TestChangesTrieStorage, - }, InMemoryBackend, overlayed_changes::OverlayedValue, + }, InMemoryBackend, overlayed_changes::{OverlayedValue, OverlayedChangeSet}, }; use sp_core::storage::{Storage, StorageChild}; + use crate::transaction_layers::{Layers, LayerEntry, COMMITTED_LAYER}; type TestBackend = InMemoryBackend; type TestExt<'a> = Ext<'a, Blake2Hasher, u64, TestBackend>; @@ -575,17 +596,26 @@ mod tests { fn prepare_overlay_with_changes() -> OverlayedChanges { OverlayedChanges { - prospective: vec![ - (EXTRINSIC_INDEX.to_vec(), OverlayedValue { - value: Some(3u32.encode()), - extrinsics: Some(vec![1].into_iter().collect()) - }), - (vec![1], OverlayedValue { - value: Some(vec![100].into_iter().collect()), - extrinsics: Some(vec![1].into_iter().collect()) - }), - ].into_iter().collect(), - committed: Default::default(), + changes: OverlayedChangeSet { + states: Default::default(), + children: Default::default(), + top: vec![ + (EXTRINSIC_INDEX.to_vec(), Layers::from_iter(vec![ + (OverlayedValue { + value: Some(3u32.encode()), + extrinsics: Some(vec![1].into_iter().collect()) + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + (vec![1], Layers::from_iter(vec![ + (OverlayedValue { + value: Some(vec![100]), + extrinsics: Some(vec![1].into_iter().collect()) + }, COMMITTED_LAYER), + ].into_iter().map(|(value, index)| LayerEntry { value, index }))), + ].into_iter().collect(), + }, + operation_from_last_gc: 0, + not_eager_gc: false, collect_extrinsics: true, stats: Default::default(), } @@ -634,7 +664,7 @@ mod tests { fn storage_changes_root_is_some_when_extrinsic_changes_are_empty() { let mut overlay = prepare_overlay_with_changes(); let mut cache = StorageTransactionCache::default(); - overlay.prospective.top.get_mut(&vec![1]).unwrap().value = None; + overlay.set_storage(vec![1], None); let storage = TestChangesTrieStorage::with_blocks(vec![(99, Default::default())]); let state = Some(ChangesTrieState::new(changes_trie_config(), Zero::zero(), &storage)); let backend = TestBackend::default(); @@ -660,7 +690,7 @@ mod tests { children: map![] }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_storage_key(&[5]), Some(vec![10])); @@ -676,7 +706,7 @@ mod tests { drop(ext); overlay.set_storage(vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_storage_key(&[40]), Some(vec![50])); @@ -709,7 +739,7 @@ mod tests { }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_backend < next_overlay assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[5]), Some(vec![10])); @@ -725,7 +755,7 @@ mod tests { drop(ext); overlay.set_child_storage(child().as_ref().to_vec(), CHILD_INFO_1, vec![50], Some(vec![50])); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); // next_overlay exist but next_backend doesn't exist assert_eq!(ext.next_child_storage_key(child(), CHILD_INFO_1, &[40]), Some(vec![50])); @@ -752,7 +782,7 @@ mod tests { ], }.into(); - let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); + let mut ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.child_storage(child(), CHILD_INFO_1, &[10]), Some(vec![10])); assert_eq!( diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 9a2dc52cca20a..9a9263a0ab12d 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -26,7 +26,6 @@ use sp_core::{ storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay, traits::{CodeExecutor, CallInWasmExt, RuntimeCode}, }; -use overlayed_changes::OverlayedChangeSet; use sp_externalities::Extensions; pub mod backend; @@ -37,10 +36,11 @@ mod ext; mod testing; mod basic; mod overlayed_changes; +mod transaction_layers; mod proving_backend; +mod stats; mod trie_backend; mod trie_backend_essence; -mod stats; pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; pub use testing::TestExternalities; @@ -73,6 +73,8 @@ pub use trie_backend_essence::{TrieBackendStorage, Storage}; pub use trie_backend::TrieBackend; pub use error::{Error, ExecutionError}; pub use in_memory_backend::InMemory as InMemoryBackend; +#[cfg(feature = "test-helpers")] +pub use transaction_layers::fuzz as transaction_layers_fuzz; pub use stats::{UsageInfo, UsageUnit, StateMachineStats}; pub use sp_core::traits::CloneableSpawn; @@ -327,7 +329,6 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where fn execute_call_with_both_strategy( &mut self, mut native_call: Option, - orig_prospective: OverlayedChangeSet, on_consensus_failure: Handler, ) -> CallResult where @@ -338,10 +339,16 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where CallResult, ) -> CallResult { + let orig_committed = self.overlay.changes.states.committed(); + // disable eager gc to be able to rollback + self.overlay.not_eager_gc = true; let (result, was_native) = self.execute_aux(true, native_call.take()); + self.overlay.not_eager_gc = false; if was_native { - self.overlay.prospective = orig_prospective.clone(); + if result.is_ok() { + self.overlay.changes.states.unchecked_rollback_committed(orig_committed); + } let (wasm_result, _) = self.execute_aux( false, native_call, @@ -363,7 +370,6 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where fn execute_call_with_native_else_wasm_strategy( &mut self, mut native_call: Option, - orig_prospective: OverlayedChangeSet, ) -> CallResult where R: Decode + Encode + PartialEq, @@ -377,7 +383,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where if !was_native || result.is_ok() { result } else { - self.overlay.prospective = orig_prospective.clone(); + self.overlay.discard_prospective(); let (wasm_result, _) = self.execute_aux( false, native_call, @@ -412,20 +418,17 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where self.overlay.set_collect_extrinsics(changes_tries_enabled); let result = { - let orig_prospective = self.overlay.prospective.clone(); match manager { ExecutionManager::Both(on_consensus_failure) => { self.execute_call_with_both_strategy( native_call.take(), - orig_prospective, on_consensus_failure, ) }, ExecutionManager::NativeElseWasm => { self.execute_call_with_native_else_wasm_strategy( native_call.take(), - orig_prospective, ) }, ExecutionManager::AlwaysWasm(trust_level) => { @@ -744,7 +747,6 @@ where mod tests { use std::collections::BTreeMap; use codec::Encode; - use overlayed_changes::OverlayedValue; use super::*; use super::ext::Ext; use super::changes_trie::Configuration as ChangesTrieConfig; @@ -962,17 +964,14 @@ mod tests { ]; let mut state = InMemoryBackend::::from(initial); let backend = state.as_trie_backend().unwrap(); - let mut overlay = OverlayedChanges { - committed: map![ - b"aba".to_vec() => OverlayedValue::from(Some(b"1312".to_vec())), - b"bab".to_vec() => OverlayedValue::from(Some(b"228".to_vec())) + let mut overlay = OverlayedChanges::new_from_top(vec![ + (b"aba".to_vec(), Some(b"1312".to_vec())), + (b"bab".to_vec(), Some(b"228".to_vec())), ], - prospective: map![ - b"abd".to_vec() => OverlayedValue::from(Some(b"69".to_vec())), - b"bbd".to_vec() => OverlayedValue::from(Some(b"42".to_vec())) - ], - ..Default::default() - }; + vec![ + (b"abd".to_vec(), Some(b"69".to_vec())), + (b"bbd".to_vec(), Some(b"42".to_vec())), + ], false); { let mut cache = StorageTransactionCache::default(); @@ -987,8 +986,11 @@ mod tests { } overlay.commit_prospective(); + overlay.discard_prospective(); + let values: HashMap<_, _> = overlay.changes.iter_values(None) + .map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))).collect(); assert_eq!( - overlay.committed, + values, map![ b"abc".to_vec() => None.into(), b"abb".to_vec() => None.into(), diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index ab50c61391bdb..b3ef8dd2c5dbc 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -30,7 +30,9 @@ use std::iter::FromIterator; use std::collections::{HashMap, BTreeMap, BTreeSet}; use codec::{Decode, Encode}; use sp_core::storage::{well_known_keys::EXTRINSIC_INDEX, OwnedChildInfo, ChildInfo}; -use std::{mem, ops}; +use crate::transaction_layers::{Layers, LayerEntry}; +use crate::transaction_layers::{States, DEFAULT_GC_CONF}; +use std::ops; use hash_db::Hasher; @@ -52,14 +54,21 @@ pub type ChildStorageCollection = Vec<(StorageKey, StorageCollection)>; /// that can be cleared. #[derive(Debug, Default, Clone)] pub struct OverlayedChanges { - /// Changes that are not yet committed. - pub(crate) prospective: OverlayedChangeSet, - /// Committed changes. - pub(crate) committed: OverlayedChangeSet, + /// Changes with their history. + pub(crate) changes: OverlayedChangeSet, + /// True if extrinsiscs stats must be collected. /// True if extrinsics stats must be collected. pub(crate) collect_extrinsics: bool, /// Collect statistic on this execution. pub(crate) stats: StateMachineStats, + /// Counter of number of operation between garbage collection. + /// Changing or deleting a value increment this counter by one. + /// An additional cost for data added is added for every n bytes of data. + /// n is currently defined by `DEFAULT_GC_CONF`. + pub(crate) operation_from_last_gc: usize, + /// Can be set to false to ensure we keep pre committed state (in case we want to move + /// back committed cursor). + pub(crate) not_eager_gc: bool, } /// The storage value, used inside OverlayedChanges. @@ -73,14 +82,23 @@ pub struct OverlayedValue { pub extrinsics: Option>, } -/// Prospective or committed overlayed change set. -#[derive(Debug, Default, Clone)] +type TreeChangeSet = BTreeMap>; + +/// Overlayed change set, content keep trace of its history. +/// +/// Maps containing a linear history of each values are used. +#[derive(Debug, Clone, Default)] #[cfg_attr(test, derive(PartialEq))] pub struct OverlayedChangeSet { + /// Indexed state for transaction layers + pub(crate) states: States, /// Top level storage changes. - pub top: BTreeMap, + pub(crate) top: TreeChangeSet, /// Child storage changes. - pub children: HashMap, OwnedChildInfo)>, + /// OwnedChildInfo is currently a non modifiable value, + /// for future child trie operations (eg full deletion) + /// it will need to manage history too (`Layers`). + pub(crate) children: HashMap, } /// A storage changes structure that can be generated by the data collected in [`OverlayedChanges`]. @@ -172,10 +190,73 @@ impl Default for StorageChanges #[cfg(test)] impl FromIterator<(StorageKey, OverlayedValue)> for OverlayedChangeSet { fn from_iter>(iter: T) -> Self { - Self { - top: iter.into_iter().collect(), - children: Default::default(), + let mut result = OverlayedChangeSet::default(); + result.top = iter.into_iter().map(|(k, value)| (k, { + let mut history = Layers::default(); + history.push_unchecked(LayerEntry { + value, + index: crate::transaction_layers::COMMITTED_LAYER, + }); + history + })).collect(); + result + } +} + +/// Variant of historical data `set` function with internal extrinsics update. +/// It avoids accessing two times the historical value item. +/// It does remove latest historical dropped items. +fn set_with_extrinsic_overlayed_value( + history: &mut Layers, + states: &States, + value: Option, + extrinsic_index: Option, +) { + if let Some(extrinsic) = extrinsic_index { + set_with_extrinsic_inner_overlayed_value(history, states, value, extrinsic) + } else { + history.set(states, OverlayedValue { + value, + extrinsics: None, + }) + } +} + +fn set_with_extrinsic_inner_overlayed_value( + history: &mut Layers, + states: &States, + value: Option>, + extrinsic_index: u32, +) { + let state = states.num_states() - 1; + if let Some(current) = history.get_mut(states) { + if current.index == state { + current.value.value = value; + current.value.extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); + } else { + let mut extrinsics = current.value.extrinsics.clone(); + extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); + history.push_unchecked(LayerEntry { + index: state, + value: OverlayedValue { + value, + extrinsics, + }, + }); } + } else { + let mut extrinsics: Option> = None; + extrinsics.get_or_insert_with(Default::default) + .insert(extrinsic_index); + history.push_unchecked(LayerEntry { + index: state, + value: OverlayedValue { + value, + extrinsics, + }, + }); } } @@ -185,17 +266,168 @@ impl OverlayedChangeSet { self.top.is_empty() && self.children.is_empty() } - /// Clear the change set. - pub fn clear(&mut self) { - self.top.clear(); - self.children.clear(); + /// Discard prospective changes from the change set state. + pub fn discard_prospective(&mut self) { + self.states.discard_prospective(); + } + + /// Commit prospective changes into the change set state. + pub fn commit_prospective(&mut self) { + self.states.commit_prospective(); + } + + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.states.start_transaction(); + } + + /// Discard a transactional layer state. + /// There is always a transactional layer running + /// (discarding the last trasactional layer open a new one). + pub fn discard_transaction(&mut self) { + self.states.discard_transaction(); + } + + /// Commit a transactional layer into previous transaction layer. + pub fn commit_transaction(&mut self) { + self.states.commit_transaction(); + } + + /// Garbage collect. + fn gc(&mut self, prune_commit: bool) { + let states = &self.states; + // retain does change values + retain(&mut self.top, |_, history| history.get_mut_pruning(states, prune_commit).is_some()); + self.children.retain(|_, (map, _child_info)| { + retain(map, |_, history| history.get_mut_pruning(states, prune_commit).is_some()); + !map.is_empty() + }); + } + + /// Iterator over current values for a given overlay, including change trie information. + pub fn iter_overlay( + &self, + storage_key: Option<&[u8]>, + ) -> (impl Iterator, Option) { + let (option_map, option_child_info) = if let Some(storage_key) = storage_key.as_ref() { + if let Some((content, child_info)) = self.children.get(*storage_key) { + (Some(content), Some(child_info.as_ref())) + } else { + (None, None) + } + } else { + (Some(&self.top), None) + }; + (option_map + .into_iter() + .flat_map(move |map| map.iter() + .filter_map(move |(k, v)| + v.get(self.states.as_ref()).0.map(|v| (k.as_slice(), v))) + ), option_child_info) + + } + + /// Iterator over current values for a given overlay. + pub fn iter_values( + &self, + storage_key: Option<&[u8]>, + ) -> impl Iterator)> { + self.iter_overlay(storage_key).0 + .map(|(k, v)| (k, v.value.as_ref().map(|v| v.as_slice()))) + } + + /// Iterator over current values of all children overlays. + pub fn children_iter( + &self, + ) -> impl Iterator)>, + ChildInfo, + )> { + self.children.iter() + .map(move |(keyspace, child_content)| ( + keyspace.as_slice(), + child_content.0.iter() + .filter_map(move |(k, v)| v.get(self.states.as_ref()).0.map(|v| ( + k.as_slice(), + v.value.as_ref() + .map(|v| v.as_slice()), + ))), + child_content.1.as_ref(), + )) + } + + /// Iterator over current values of all children overlays. + /// This is a variant of `children_iter` with owned `Vec` keys and values. + pub fn owned_children_iter<'a>( + &'a self, + ) -> impl Iterator, + impl Iterator, Option>)> + 'a, + OwnedChildInfo, + )> + 'a { + self.children.iter() + .map(move |(keyspace, (map, child_info))| ( + keyspace.to_vec(), + map.iter() + .filter_map(move |(k, v)| + v.get(self.states.as_ref()).0 + .map(|v| (k.to_vec(), v.value.as_ref().map(|v| v.to_vec())))), + child_info.to_owned(), + )) + } + + /// Iterator over current values of all children overlays, including change trie information. + pub fn children_iter_overlay( + &self, + ) -> impl Iterator, + ChildInfo, + )> { + self.children.iter() + .map(move |(keyspace, (map, child_info))| ( + keyspace.as_slice(), + map.iter() + .filter_map(move |(k, v)| + v.get(self.states.as_ref()).0.map(|v| (k.as_slice(), v))), + child_info.as_ref(), + )) + } + + /// Test only method for accessing current prospective values. + /// This method is only here to keep compatibility with previous tests, + /// please do not use for new tests. + #[cfg(test)] + pub(crate) fn top_prospective(&self) -> BTreeMap, OverlayedValue> { + let mut result = BTreeMap::new(); + for (k, v) in self.top.iter() { + if let Some(v) = v.get_prospective(&self.states) { + result.insert(k.clone(), v.clone()); + } + } + result + } + + /// Test only method to access current committed changes. + /// This method is only here to keep compatibility with previous tests, + /// please do not use for new tests. + #[cfg(test)] + pub(crate) fn top_committed(&self) -> BTreeMap, OverlayedValue> { + let mut result = BTreeMap::new(); + for (k, v) in self.top.iter() { + if let Some(v) = v.get_committed(&self.states) { + result.insert(k.clone(), v.clone()); + } + } + result } } impl OverlayedChanges { /// Whether the overlayed changes are empty. pub fn is_empty(&self) -> bool { - self.prospective.is_empty() && self.committed.is_empty() + self.changes.is_empty() } /// Ask to collect/not to collect extrinsics indices where key(s) has been changed. @@ -203,271 +435,275 @@ impl OverlayedChanges { self.collect_extrinsics = collect_extrinsics; } - /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred - /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose - /// value has been set. - pub fn storage(&self, key: &[u8]) -> Option> { - self.prospective.top.get(key) - .or_else(|| self.committed.top.get(key)) - .map(|x| { - let size_read = x.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); + /// Same as `child_storage` but does not update value. It should + /// consequently be used only when the query is bounded to a bounded + /// number of call before memory get drop. + /// There returned boolean indicates if there is a query overhead. + pub fn storage_once(&self, key: &[u8]) -> Option<(Option<&[u8]>, bool)> { + if let Some(overlay_value) = self.changes.top.get(key) { + if let (Some(o_value), tail) = overlay_value.get(self.changes.states.as_ref()) { + let size_read = o_value.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_read_modified(size_read); - x.value.as_ref().map(AsRef::as_ref) - }) + return Some((o_value.value.as_ref().map(|v| v.as_slice()), tail)) + } + } + None + } + + /// Same as `child_storage` but does not update value. It should + /// consequently be used only when the query is bounded to a bounded + /// number of call before memory get drop. + /// There returned boolean indicates if there is a query overhead. + pub fn child_storage_once( + &self, + storage_key: &[u8], + key: &[u8], + ) -> Option<(Option<&[u8]>, bool)> { + if let Some(map) = self.changes.children.get(storage_key) { + if let Some(overlay_value) = map.0.get(key) { + if let (Some(o_value), tail) = overlay_value.get(self.changes.states.as_ref()) { + let size_read = o_value.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); + self.stats.tally_read_modified(size_read); + return Some((o_value.value.as_ref().map(|v| v.as_slice()), tail)) + } + } + } + None } /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose /// value has been set. - pub fn child_storage(&self, storage_key: &[u8], key: &[u8]) -> Option> { - if let Some(map) = self.prospective.children.get(storage_key) { - if let Some(val) = map.0.get(key) { - let size_read = val.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); + pub fn storage(&mut self, key: &[u8]) -> Option> { + if let Some(overlay_value) = self.changes.top.get_mut(key) { + if let Some(o_value) = overlay_value.get_mut(&self.changes.states) { + let size_read = o_value.value.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_read_modified(size_read); - return Some(val.value.as_ref().map(AsRef::as_ref)); + return Some(o_value.value.value.as_ref().map(|v| v.as_slice())) } } + None + } - if let Some(map) = self.committed.children.get(storage_key) { - if let Some(val) = map.0.get(key) { - let size_read = val.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); - self.stats.tally_read_modified(size_read); - return Some(val.value.as_ref().map(AsRef::as_ref)); + /// Returns a double-Option: None if the key is unknown (i.e. and the query should be referred + /// to the backend); Some(None) if the key has been deleted. Some(Some(...)) for a key whose + /// value has been set. + pub fn child_storage(&mut self, storage_key: &[u8], key: &[u8]) -> Option> { + if let Some(map) = self.changes.children.get_mut(storage_key) { + if let Some(overlay_value) = map.0.get_mut(key) { + if let Some(o_value) = overlay_value.get_mut(&self.changes.states) { + let size_read = o_value.value.value.as_ref().map(|x| x.len() as u64).unwrap_or(0); + self.stats.tally_read_modified(size_read); + return Some(o_value.value.value.as_ref().map(|v| v.as_slice())) + } } } - None } /// Inserts the given key-value pair into the prospective change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_storage(&mut self, key: StorageKey, val: Option) { - let size_write = val.as_ref().map(|x| x.len() as u64).unwrap_or(0); + pub fn set_storage(&mut self, key: StorageKey, value: Option) { + let size_write = value.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_write_overlay(size_write); + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); - let entry = self.prospective.top.entry(key).or_default(); - entry.value = val; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } + let entry = self.changes.top.entry(key).or_default(); + set_with_extrinsic_overlayed_value( + entry, + &self.changes.states, + value, + extrinsic_index, + ); } /// Inserts the given key-value pair into the prospective child change set. /// /// `None` can be used to delete a value specified by the given key. - pub(crate) fn set_child_storage( + pub fn set_child_storage( &mut self, storage_key: StorageKey, child_info: ChildInfo, key: StorageKey, - val: Option, + value: Option, ) { - let size_write = val.as_ref().map(|x| x.len() as u64).unwrap_or(0); + let size_write = value.as_ref().map(|x| x.len() as u64).unwrap_or(0); self.stats.tally_write_overlay(size_write); + self.operation_from_last_gc += DEFAULT_GC_CONF.operation_cost(value.as_ref()); let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key) + let map_entry = self.changes.children.entry(storage_key) .or_insert_with(|| (Default::default(), child_info.to_owned())); let updatable = map_entry.1.try_update(child_info); debug_assert!(updatable); let entry = map_entry.0.entry(key).or_default(); - entry.value = val; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } + set_with_extrinsic_overlayed_value( + entry, + &self.changes.states, + value, + extrinsic_index, + ); } /// Clear child storage of given storage key. /// /// NOTE that this doesn't take place immediately but written into the prospective - /// change set, and still can be reverted by [`discard_prospective`]. + /// change set, and still can be reverted by [`discard_prospective`] + /// or [`discard_transaction`] /// /// [`discard_prospective`]: #method.discard_prospective - pub(crate) fn clear_child_storage( + /// [`discard_transaction`]: #method.discard_transaction + pub fn clear_child_storage( &mut self, storage_key: &[u8], child_info: ChildInfo, ) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key.to_vec()) + let states = &self.changes.states; + let map_entry = self.changes.children.entry(storage_key.to_vec()) .or_insert_with(|| (Default::default(), child_info.to_owned())); + self.operation_from_last_gc += map_entry.0.len(); let updatable = map_entry.1.try_update(child_info); debug_assert!(updatable); - map_entry.0.values_mut().for_each(|e| { - if let Some(extrinsic) = extrinsic_index { - e.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } - - e.value = None; - }); - - if let Some((committed_map, _child_info)) = self.committed.children.get(storage_key) { - for (key, value) in committed_map.iter() { - if !map_entry.0.contains_key(key) { - map_entry.0.insert(key.clone(), OverlayedValue { - value: None, - extrinsics: extrinsic_index.map(|i| { - let mut e = value.extrinsics.clone() - .unwrap_or_else(|| BTreeSet::default()); - e.insert(i); - e - }), - }); - } - } - } + map_entry.0.values_mut() + .for_each(|e| set_with_extrinsic_overlayed_value(e, states, None, extrinsic_index)); } /// Removes all key-value pairs which keys share the given prefix. /// /// NOTE that this doesn't take place immediately but written into the prospective - /// change set, and still can be reverted by [`discard_prospective`]. + /// change set, and still can be reverted by [`discard_prospective`] + /// or [`discard_transaction`] /// /// [`discard_prospective`]: #method.discard_prospective - pub(crate) fn clear_prefix(&mut self, prefix: &[u8]) { + /// [`discard_transaction`]: #method.discard_transaction + pub fn clear_prefix(&mut self, prefix: &[u8]) { let extrinsic_index = self.extrinsic_index(); - // Iterate over all prospective and mark all keys that share - // the given prefix as removed (None). - for (key, entry) in self.prospective.top.iter_mut() { - if key.starts_with(prefix) { - entry.value = None; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } - } - } - - // Then do the same with keys from committed changes. - // NOTE that we are making changes in the prospective change set. - for key in self.committed.top.keys() { + let mut number_removed = 0; + for (key, entry) in self.changes.top.iter_mut() { if key.starts_with(prefix) { - let entry = self.prospective.top.entry(key.clone()).or_default(); - entry.value = None; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } + number_removed += 1; + set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } + self.operation_from_last_gc += number_removed; } - pub(crate) fn clear_child_prefix( + /// Removes all key-value pairs which keys share the given prefix for + /// a given child trie. + /// + /// NOTE that this doesn't take place immediately but written into the prospective + /// change set, and still can be reverted by [`discard_prospective`] + /// or [`discard_transaction`] + /// + /// [`discard_prospective`]: #method.discard_prospective + /// [`discard_transaction`]: #method.discard_transaction + pub fn clear_child_prefix( &mut self, storage_key: &[u8], child_info: ChildInfo, prefix: &[u8], ) { let extrinsic_index = self.extrinsic_index(); - let map_entry = self.prospective.children.entry(storage_key.to_vec()) - .or_insert_with(|| (Default::default(), child_info.to_owned())); - let updatable = map_entry.1.try_update(child_info); - debug_assert!(updatable); - - for (key, entry) in map_entry.0.iter_mut() { - if key.starts_with(prefix) { - entry.value = None; + if let Some(child_change) = self.changes.children.get_mut(storage_key) { + let mut number_removed = 0; + let updatable = child_change.1.try_update(child_info); + debug_assert!(updatable); - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } - } - } - - if let Some((child_committed, _child_info)) = self.committed.children.get(storage_key) { - // Then do the same with keys from committed changes. - // NOTE that we are making changes in the prospective change set. - for key in child_committed.keys() { + for (key, entry) in child_change.0.iter_mut() { if key.starts_with(prefix) { - let entry = map_entry.0.entry(key.clone()).or_default(); - entry.value = None; - - if let Some(extrinsic) = extrinsic_index { - entry.extrinsics.get_or_insert_with(Default::default) - .insert(extrinsic); - } + number_removed += 1; + set_with_extrinsic_overlayed_value(entry, &self.changes.states, None, extrinsic_index); } } + self.operation_from_last_gc += number_removed; } } /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { - self.prospective.clear(); + self.changes.discard_prospective(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { + self.operation_from_last_gc = 0; + self.gc(!self.not_eager_gc); + } } /// Commit prospective changes to state. pub fn commit_prospective(&mut self) { - if self.committed.is_empty() { - mem::swap(&mut self.prospective, &mut self.committed); - } else { - let top_to_commit = mem::replace(&mut self.prospective.top, BTreeMap::new()); - for (key, val) in top_to_commit.into_iter() { - let entry = self.committed.top.entry(key).or_default(); - entry.value = val.value; - - if let Some(prospective_extrinsics) = val.extrinsics { - entry.extrinsics.get_or_insert_with(Default::default) - .extend(prospective_extrinsics); - } - } - for (storage_key, (map, child_info)) in self.prospective.children.drain() { - let child_content = self.committed.children.entry(storage_key) - .or_insert_with(|| (Default::default(), child_info)); - // No update to child info at this point (will be needed for deletion). - for (key, val) in map.into_iter() { - let entry = child_content.0.entry(key).or_default(); - entry.value = val.value; - - if let Some(prospective_extrinsics) = val.extrinsics { - entry.extrinsics.get_or_insert_with(Default::default) - .extend(prospective_extrinsics); - } - } - } + self.changes.commit_prospective(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_commit_gc { + self.operation_from_last_gc = 0; + self.gc(!self.not_eager_gc); + } + } + + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.changes.start_transaction(); + } + + /// Discard a transactional layer. + /// A transaction is always running (history always end with pending). + pub fn discard_transaction(&mut self) { + self.changes.discard_transaction(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { + self.operation_from_last_gc = 0; + self.gc(false); + } + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + self.changes.commit_transaction(); + if self.operation_from_last_gc > DEFAULT_GC_CONF.trigger_transaction_gc { + self.operation_from_last_gc = 0; + self.gc(false); } } /// Consume `OverlayedChanges` and take committed set. - /// - /// Panics: - /// Will panic if there are any uncommitted prospective changes. - fn drain_committed(&mut self) -> ( + pub fn drain_committed(&mut self) -> ( impl Iterator)>, - impl Iterator)>, OwnedChildInfo))>, + impl Iterator)>, + OwnedChildInfo, + )>, ) { - assert!(self.prospective.is_empty()); + let top = std::mem::replace(&mut self.changes.top, Default::default()); + let children = std::mem::replace(&mut self.changes.children, Default::default()); + let committed = self.changes.states.committed(); + let states = self.changes.states.clone(); + let states2 = std::mem::replace(&mut self.changes.states, Default::default()); ( - std::mem::replace(&mut self.committed.top, Default::default()) - .into_iter() - .map(|(k, v)| (k, v.value)), - std::mem::replace(&mut self.committed.children, Default::default()) - .into_iter() - .map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))), + top.into_iter() + .filter_map(move |(k, v)| v.into_committed(states.as_ref(), committed) + .map(|v| (k, v.value))), + children.into_iter().map(move |(storage_key, child_content)| { + let states2 = states2.clone(); + (storage_key, child_content.0.into_iter() + .filter_map(move |(k, v)| v.into_committed(states2.as_ref(), committed) + .map(|v| (k, v.value))), + child_content.1.to_owned()) + }) ) } /// Convert this instance with all changes into a [`StorageChanges`] instance. - pub fn into_storage_changes< - B: Backend, H: Hasher, N: BlockNumber - >( + pub fn into_storage_changes, H: Hasher, N: BlockNumber>( mut self, backend: &B, changes_trie_state: Option<&ChangesTrieState>, parent_hash: H::Out, mut cache: StorageTransactionCache, - ) -> Result, String> where H::Out: Ord + Encode + 'static { + ) -> Result, String> + where + H::Out: Ord + Encode + 'static, + { self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache) } @@ -507,7 +743,9 @@ impl OverlayedChanges { Ok(StorageChanges { main_storage_changes: main_storage_changes.collect(), - child_storage_changes: child_storage_changes.map(|(sk, it)| (sk, it.0.collect())).collect(), + child_storage_changes: child_storage_changes.map(|(sk, it, _child_info)| + (sk, it.collect()) + ).collect(), transaction, transaction_storage_root, changes_trie_transaction, @@ -517,10 +755,29 @@ impl OverlayedChanges { /// Inserts storage entry responsible for current extrinsic index. #[cfg(test)] pub(crate) fn set_extrinsic_index(&mut self, extrinsic_index: u32) { - self.prospective.top.insert(EXTRINSIC_INDEX.to_vec(), OverlayedValue { - value: Some(extrinsic_index.encode()), - extrinsics: None, - }); + self.set_storage(EXTRINSIC_INDEX.to_vec(), Some(extrinsic_index.encode())); + } + + /// Test only method to create a change set from committed + /// and prospective content. + #[cfg(test)] + pub(crate) fn new_from_top( + committed: Vec<(Vec, Option>)>, + prospective: Vec<(Vec, Option>)>, + collect_extrinsics: bool, + ) -> Self { + let changes = OverlayedChangeSet::default(); + let mut result = OverlayedChanges { + changes, + operation_from_last_gc: 0, + not_eager_gc: false, + collect_extrinsics, + stats: Default::default(), + }; + committed.into_iter().for_each(|(k, v)| result.set_storage(k, v)); + result.changes.commit_prospective(); + prospective.into_iter().for_each(|(k, v)| result.set_storage(k, v)); + result } /// Returns current extrinsic index to use in changes trie construction. @@ -529,7 +786,7 @@ impl OverlayedChanges { /// set this index before first and unset after last extrinsic is executed. /// Changes that are made outside of extrinsics, are marked with /// `NO_EXTRINSIC_INDEX` index. - fn extrinsic_index(&self) -> Option { + fn extrinsic_index(&mut self) -> Option { match self.collect_extrinsics { true => Some( self.storage(EXTRINSIC_INDEX) @@ -539,6 +796,35 @@ impl OverlayedChanges { } } + #[cfg(any(test, feature = "test-helpers"))] + /// Iterator over current values of the overlay. + pub fn iter_values( + &self, + storage_key: Option<&[u8]>, + ) -> impl Iterator)> { + self.changes.iter_values(storage_key) + } + + #[cfg(any(test, feature = "test-helpers"))] + /// Count the number of pending key value pairs. + /// This is slow and should only be use for debugging or testing. + pub fn top_count_keyvalue_pair(&self) -> usize { + let mut result = 0; + for (_, v) in self.changes.top.iter() { + result += v.len() + } + result + } + + /// Garbage collection of unneeded memory from + /// every `Layers` of values in regard to current transactional state. + /// If `eager` parameter is true, the garbage collection will go past + /// the latest discarded and committed transaction and reclaim the maximal + /// size of memory at an additional computational cost. + pub fn gc(&mut self, eager: bool) { + self.changes.gc(eager); + } + /// Generate the storage root using `backend` and all changes from `prospective` and `committed`. /// /// Returns the storage root and caches storage transaction in the given `cache`. @@ -547,32 +833,15 @@ impl OverlayedChanges { backend: &B, cache: &mut StorageTransactionCache, ) -> H::Out - where H::Out: Ord + Encode, + where + H::Out: Ord + Encode, { - let child_storage_keys = self.prospective.children.keys() - .chain(self.committed.children.keys()); - let child_delta_iter = child_storage_keys.map(|storage_key| - ( - storage_key.clone(), - self.committed.children.get(storage_key) - .into_iter() - .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) - .chain( - self.prospective.children.get(storage_key) - .into_iter() - .flat_map(|(map, _)| map.iter().map(|(k, v)| (k.clone(), v.value.clone()))) - ), - self.child_info(storage_key).cloned() - .expect("child info initialized in either committed or prospective"), - ) - ); + let child_delta_iter = self.changes.owned_children_iter(); // compute and memoize - let delta = self.committed.top.iter().map(|(k, v)| (k.clone(), v.value.clone())) - .chain(self.prospective.top.iter().map(|(k, v)| (k.clone(), v.value.clone()))); + let delta = self.changes.iter_values(None).map(|(k, v)| (k.to_vec(), v.map(|s| s.to_vec()))); let (root, transaction) = backend.full_storage_root(delta, child_delta_iter); - cache.transaction = Some(transaction); cache.transaction_storage_root = Some(root); @@ -593,7 +862,10 @@ impl OverlayedChanges { parent_hash: H::Out, panic_on_storage_error: bool, cache: &mut StorageTransactionCache, - ) -> Result, ()> where H::Out: Ord + Encode + 'static { + ) -> Result, ()> + where + H::Out: Ord + Encode + 'static, + { build_changes_trie::<_, H, N>( backend, changes_trie_state, @@ -610,65 +882,73 @@ impl OverlayedChanges { /// Get child info for a storage key. /// Take the latest value so prospective first. - pub fn child_info(&self, storage_key: &[u8]) -> Option<&OwnedChildInfo> { - if let Some((_, ci)) = self.prospective.children.get(storage_key) { - return Some(&ci); - } - if let Some((_, ci)) = self.committed.children.get(storage_key) { - return Some(&ci); + pub fn child_info(&self, storage_key: &[u8]) -> Option { + if let Some((_, child_info)) = self.changes.children.get(storage_key) { + return Some(child_info.as_ref()); } None } /// Returns the next (in lexicographic order) storage key in the overlayed alongside its value. /// If no value is next then `None` is returned. - pub fn next_storage_key_change(&self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { + pub fn next_storage_key_change(&mut self, key: &[u8]) -> Option<(&[u8], &OverlayedValue)> { let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded); - let next_prospective_key = self.prospective.top - .range::<[u8], _>(range) - .next() - .map(|(k, v)| (&k[..], v)); - - let next_committed_key = self.committed.top - .range::<[u8], _>(range) - .next() - .map(|(k, v)| (&k[..], v)); - - match (next_committed_key, next_prospective_key) { - // Committed is strictly less than prospective - (Some(committed_key), Some(prospective_key)) if committed_key.0 < prospective_key.0 => - Some(committed_key), - (committed_key, None) => committed_key, - // Prospective key is less or equal to committed or committed doesn't exist - (_, prospective_key) => prospective_key, + let mut next_keys = self.changes.top.range_mut::<[u8], _>(range); + while let Some((key, historic_value)) = next_keys.next() { + if let Some(overlay_value) = historic_value.get_mut(&self.changes.states) { + return Some((key, overlay_value.value)); + } } + None } /// Returns the next (in lexicographic order) child storage key in the overlayed alongside its /// value. If no value is next then `None` is returned. pub fn next_child_storage_key_change( - &self, + &mut self, storage_key: &[u8], key: &[u8] ) -> Option<(&[u8], &OverlayedValue)> { let range = (ops::Bound::Excluded(key), ops::Bound::Unbounded); - let next_prospective_key = self.prospective.children.get(storage_key) - .and_then(|(map, _)| map.range::<[u8], _>(range).next().map(|(k, v)| (&k[..], v))); - - let next_committed_key = self.committed.children.get(storage_key) - .and_then(|(map, _)| map.range::<[u8], _>(range).next().map(|(k, v)| (&k[..], v))); + if let Some(child) = self.changes.children.get_mut(storage_key) { + let mut next_keys = child.0.range_mut::<[u8], _>(range); + while let Some((key, historic_value)) = next_keys.next() { + if let Some(overlay_value) = historic_value.get_mut(&self.changes.states) { + return Some((key, overlay_value.value)); + } + } + } + None + } +} - match (next_committed_key, next_prospective_key) { - // Committed is strictly less than prospective - (Some(committed_key), Some(prospective_key)) if committed_key.0 < prospective_key.0 => - Some(committed_key), - (committed_key, None) => committed_key, - // Prospective key is less or equal to committed or committed doesn't exist - (_, prospective_key) => prospective_key, +/// This is an implementation of retain for btreemap, +/// this is used to keep a similar api with previous +/// hashmap usage. +/// This could also be easilly replace if btreemap gets +/// an implementation for retain in the future. +fn retain(map: &mut BTreeMap, mut f: F) + where + F: FnMut(&K, &mut V) -> bool, +{ + // this is use to discard some historical values when + // their status is cleared. + // Regarding this use case we will only remove individually. + // Rebuilding trie from iterator may be better if there is + // more removal. + // Regarding this use case skipping removal can be a better + // choice. + let mut rem = Vec::new(); + for (k, v) in map.iter_mut() { + if !f(k, v) { + rem.push(k.clone()); } } + for k in rem { + map.remove(&k); + } } #[cfg(test)] @@ -688,12 +968,11 @@ mod tests { use crate::ext::Ext; use super::*; - fn strip_extrinsic_index(map: &BTreeMap) + fn strip_extrinsic_index(mut map: BTreeMap) -> BTreeMap { - let mut clone = map.clone(); - clone.remove(&EXTRINSIC_INDEX.to_vec()); - clone + map.remove(&EXTRINSIC_INDEX.to_vec()); + map } #[test] @@ -733,18 +1012,16 @@ mod tests { (b"doug".to_vec(), b"notadog".to_vec()), ].into_iter().collect(); let backend = InMemoryBackend::::from(initial); - let mut overlay = OverlayedChanges { - committed: vec![ + let mut overlay = OverlayedChanges::new_from_top( + vec![ (b"dog".to_vec(), Some(b"puppy".to_vec()).into()), (b"dogglesworth".to_vec(), Some(b"catYYY".to_vec()).into()), (b"doug".to_vec(), Some(vec![]).into()), ].into_iter().collect(), - prospective: vec![ + vec![ (b"dogglesworth".to_vec(), Some(b"cat".to_vec()).into()), (b"doug".to_vec(), None.into()), - ].into_iter().collect(), - ..Default::default() - }; + ].into_iter().collect(), false); let mut cache = StorageTransactionCache::default(); let mut ext = Ext::new( @@ -775,7 +1052,7 @@ mod tests { overlay.set_extrinsic_index(2); overlay.set_storage(vec![1], Some(vec![6])); - assert_eq!(strip_extrinsic_index(&overlay.prospective.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_prospective()), vec![ (vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }), @@ -793,7 +1070,7 @@ mod tests { overlay.set_extrinsic_index(4); overlay.set_storage(vec![1], Some(vec![8])); - assert_eq!(strip_extrinsic_index(&overlay.committed.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_committed()), vec![ (vec![1], OverlayedValue { value: Some(vec![6]), extrinsics: Some(vec![0, 2].into_iter().collect()) }), @@ -803,17 +1080,17 @@ mod tests { extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }), ].into_iter().collect()); - assert_eq!(strip_extrinsic_index(&overlay.prospective.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_prospective()), vec![ (vec![1], OverlayedValue { value: Some(vec![8]), - extrinsics: Some(vec![4].into_iter().collect()) }), + extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }), (vec![3], OverlayedValue { value: Some(vec![7]), - extrinsics: Some(vec![3].into_iter().collect()) }), + extrinsics: Some(vec![1, 3].into_iter().collect()) }), ].into_iter().collect()); overlay.commit_prospective(); - assert_eq!(strip_extrinsic_index(&overlay.committed.top), + assert_eq!(strip_extrinsic_index(overlay.changes.top_committed()), vec![ (vec![1], OverlayedValue { value: Some(vec![8]), extrinsics: Some(vec![0, 2, 4].into_iter().collect()) }), @@ -823,10 +1100,83 @@ mod tests { extrinsics: Some(vec![NO_EXTRINSIC_INDEX].into_iter().collect()) }), ].into_iter().collect()); - assert_eq!(overlay.prospective, + assert_eq!(overlay.changes.top_prospective(), Default::default()); } + #[test] + fn overlayed_storage_transactions() { + let mut overlayed = OverlayedChanges::default(); + + let key = vec![42, 69, 169, 142]; + + assert!(overlayed.storage(&key).is_none()); + + // discard transaction similar to discard prospective if no transaction. + + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.discard_transaction(); + assert_eq!(overlayed.storage(&key), None); + + overlayed.discard_prospective(); + assert_eq!(overlayed.storage(&key), None); + + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.commit_transaction(); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.discard_transaction(); + assert_eq!(overlayed.storage(&key), None); + // basic transaction test + // tx:1 + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + + overlayed.start_transaction(); + // tx:2 + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3, 4])); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3, 4][..])); + + overlayed.start_transaction(); + // tx:3 + overlayed.set_storage(key.clone(), None); + assert_eq!(overlayed.storage(&key).unwrap(), None); + + overlayed.discard_transaction(); + // tx:2 + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3, 4][..])); + + overlayed.start_transaction(); + // tx:3 + overlayed.set_storage(key.clone(), None); + assert_eq!(overlayed.storage(&key).unwrap(), None); + + overlayed.commit_transaction(); + // tx:2 + assert_eq!(overlayed.storage(&key).unwrap(), None); + + overlayed.discard_transaction(); + // tx:1 + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + overlayed.discard_prospective(); + assert_eq!(overlayed.storage(&key), None); + + // multiple transaction end up to prospective value + overlayed.start_transaction(); + overlayed.set_storage(key.clone(), Some(vec![1])); + overlayed.start_transaction(); + overlayed.set_storage(key.clone(), Some(vec![1, 2])); + overlayed.start_transaction(); + overlayed.set_storage(key.clone(), Some(vec![1, 2, 3])); + + overlayed.commit_prospective(); + assert_eq!(overlayed.storage(&key).unwrap(), Some(&[1, 2, 3][..])); + } + #[test] fn next_storage_key_change_works() { let mut overlay = OverlayedChanges::default(); diff --git a/primitives/state-machine/src/testing.rs b/primitives/state-machine/src/testing.rs index aec42c7678707..794b2b160d145 100644 --- a/primitives/state-machine/src/testing.rs +++ b/primitives/state-machine/src/testing.rs @@ -120,18 +120,15 @@ impl TestExternalities /// Return a new backend with all pending value. pub fn commit_all(&self) -> InMemoryBackend { - let top: Vec<_> = self.overlay.committed.top.clone().into_iter() - .chain(self.overlay.prospective.top.clone().into_iter()) - .map(|(k, v)| (k, v.value)).collect(); + let top: Vec<_> = self.overlay.changes.iter_overlay(None).0 + .map(|(k, v)| (k.to_vec(), v.value.clone())).collect(); let mut transaction = vec![(None, top)]; - self.overlay.committed.children.clone().into_iter() - .chain(self.overlay.prospective.children.clone().into_iter()) - .for_each(|(keyspace, (map, child_info))| { + self.overlay.changes.children_iter_overlay() + .for_each(|(storage_key, map, child_info)| { transaction.push(( - Some((keyspace, child_info)), - map.into_iter() - .map(|(k, v)| (k, v.value)) + Some((storage_key.to_vec(), child_info.to_owned())), + map.map(|(k, v)| (k.to_vec(), v.value.clone())) .collect::>(), )) }); diff --git a/primitives/state-machine/src/transaction_layers.rs b/primitives/state-machine/src/transaction_layers.rs new file mode 100644 index 0000000000000..547a6edf69765 --- /dev/null +++ b/primitives/state-machine/src/transaction_layers.rs @@ -0,0 +1,953 @@ +// Copyright 2017-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Types and methods for managing a stack of transactional values. + + +#[derive(Debug, Clone, Eq, PartialEq, Copy)] +/// Possible states of a transactional layer. +pub(crate) enum TransactionState { + /// Data is under change and can still be dropped. + Pending, + /// Same as `Pending` but does count as a transaction start. + TxPending, + /// The transaction has been discarded. + /// Data from a `LayerEntry` pointing to this layer state should + /// not be returned and can be removed. + Dropped, +} + +/// States for all past transaction layers. +#[derive(Debug, Clone)] +#[cfg_attr(any(test, feature = "test-helpers"), derive(PartialEq))] +pub struct States { + history: Vec, + // Keep track of the latest start of transaction + // for any `LayerEntry`. + // This information is redundant as it could be + // calculated by iterating backward over `history` + // field. + // Managing this cache let us implement `transaction_start_windows` + // function in constant time. + start_transaction_window: Vec, + // `commit` stores the index of the layer for which a previous + // prospective commit occurs. + commit: usize, +} + +impl Default for States { + fn default() -> Self { + States { + history: vec![TransactionState::Pending], + start_transaction_window: vec![0], + commit: 0, + } + } +} + +impl States { + /// Get reference of states, this is enough + /// information to query a `Layers` of values. + pub(crate) fn as_ref(&self) -> &[TransactionState] { + self.history.as_ref() + } + + /// Current number of inner states. + pub(crate) fn num_states(&self) -> usize { + self.history.len() + } + + /// Get index of committed layer. + pub fn committed(&self) -> usize { + self.commit + } + + /// Allow to rollback to a previous committed + /// index. + /// This can only work if there was no eager + /// garbage collection (eager collection can + /// drop `LayersEntry` in the interval between + /// `old_committed` and `self.commit` and moving + /// back the committed cursor is therefore not + /// allowed in this case). + pub fn unchecked_rollback_committed(&mut self, old_committed: usize) { + self.commit = old_committed; + self.discard_prospective(); + } + + /// Instantiate a random state, only for testing. + #[cfg(test)] + pub(crate) fn test_vector( + history: Vec, + start_transaction_window: Vec, + commit: usize, + ) -> Self { + States { history, start_transaction_window, commit } + } + + /// Discard prospective changes. + /// This action invalidates all transaction state up to the committed index. + pub fn discard_prospective(&mut self) { + for i in self.commit .. self.history.len() { + self.history[i] = TransactionState::Dropped; + self.start_transaction_window[i] = self.commit; + } + self.history.push(TransactionState::Pending); + self.start_transaction_window.push(self.commit); + } + + /// Commit prospective changes. + /// It push the committed pointer at the very + /// end of the state, blocking discard of any + /// current state. + pub fn commit_prospective(&mut self) { + self.commit = self.history.len(); + self.history.push(TransactionState::Pending); + for i in 0..self.history.len() - 1 { + self.start_transaction_window[i] = self.commit; + } + self.start_transaction_window.push(self.commit); + } + + /// Create a new transactional layer. + pub fn start_transaction(&mut self) { + self.start_transaction_window.push(self.history.len()); + self.history.push(TransactionState::TxPending); + } + + /// Discard a transactional layer. + /// A transaction is always running, so this also append + /// a new pending transactional layer to the state. + pub fn discard_transaction(&mut self) { + let mut i = self.history.len(); + while i > self.commit { + i -= 1; + match self.history[i] { + TransactionState::Dropped => (), + TransactionState::Pending => { + self.history[i] = TransactionState::Dropped; + }, + TransactionState::TxPending => { + self.history[i] = TransactionState::Dropped; + break; + }, + } + } + self.history.push(TransactionState::Pending); + self.start_transaction_window.truncate(i); + let previous = self.start_transaction_window.last() + .cloned().unwrap_or(self.commit); + self.start_transaction_window.resize(self.history.len(), previous); + } + + /// Commit a transactional layer. + pub fn commit_transaction(&mut self) { + let latest_transaction = self.start_transaction_window[self.history.len() - 1]; + if latest_transaction > self.commit { + self.history[latest_transaction] = TransactionState::Pending; + } + self.history.push(TransactionState::Pending); + self.start_transaction_window.truncate(latest_transaction); + // self.start_transaction_window.truncate(latest_transaction); + let previous = self.start_transaction_window.last() + .cloned().unwrap_or(self.commit); + self.start_transaction_window.resize(self.history.len(), previous); + } +} + +#[inline] +/// Get previous index of pending state. +/// +/// Used to say if it is possible to drop a committed transaction +/// state value. +/// Committed index is seen as a transaction state. +pub fn transaction_start_windows(states: &States, from: usize) -> usize { + states.start_transaction_window[from] +} + +/// Stack of values at different transactional layers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Layers(pub(crate) smallvec::SmallVec<[LayerEntry; ALLOCATED_HISTORY]>); + +#[cfg(test)] +/// Index of reserved layer for commited values. +pub(crate) const COMMITTED_LAYER: usize = 0; + +/// Start transaction stack layer at a size of two, +/// to only allocate at first transaction (we always +/// have a fix committed layer). +const ALLOCATED_HISTORY: usize = 2; + +impl Default for Layers { + fn default() -> Self { + Layers(Default::default()) + } +} + +/// An value entry of a indexed transactional layer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LayerEntry { + /// The stored value. + pub value: V, + /// The transactional layer index. + pub index: usize, +} + +impl From<(V, usize)> for LayerEntry { + fn from(input: (V, usize)) -> LayerEntry { + LayerEntry { value: input.0, index: input.1 } + } +} + +impl LayerEntry { + fn as_ref(&self) -> LayerEntry<&V> { + LayerEntry { value: &self.value, index: self.index } + } +} + +impl Layers { + fn remove_until(&mut self, index: usize) { + if index > 0 { + if self.0.spilled() { + let owned = std::mem::replace(&mut self.0, Default::default()); + self.0 = smallvec::SmallVec::from_vec(owned.into_vec().split_off(index)); + } else { + for i in (0..index).rev() { + self.0.remove(i); + } + } + } + } + + fn get_pending_unchecked( + &self, + states: &[TransactionState], + history_index: usize, + ) -> Option> { + let LayerEntry { value, index } = self.0[history_index].as_ref(); + match states[index] { + TransactionState::Dropped => (), + TransactionState::Pending + | TransactionState::TxPending => + return Some(LayerEntry { value, index }), + } + None + } + + /// Push a value without checking transactional layer consistency. + pub(crate) fn push_unchecked(&mut self, item: LayerEntry) { + self.0.push(item); + } + + /// Access to the latest pending value (non dropped state). + /// When possible please prefer `get_mut` as it can free + /// some memory. + /// If tail freeing is needed, the function also return true + /// (this indicates the operation run suboptimally and could + /// in certain condition leads to bad complexity). + pub(crate) fn get(&self, states: &[TransactionState]) -> (Option<&V>, bool) { + let self_len = self.0.len(); + if self_len == 0 { + return (None, false); + } + debug_assert!(states.len() >= self_len); + for index in (0 .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states, index) { + return (Some(h.value), index != self_len - 1); + } + } + (None, true) + } + + /// Access to latest pending value (non dropped state). + /// + /// This method removes latest dropped and merged values, + /// only keeping the latest valid value. + /// + /// Returns mutable latest pending value. + pub(crate) fn get_mut( + &mut self, + states: &States, + ) -> Option> { + let self_len = self.0.len(); + if self_len == 0 { + return None; + } + debug_assert!(states.history.len() >= self_len); + let mut result = None; + let mut start_transaction_window = usize::max_value(); + let mut previous_switch = None; + for index in (0 .. self_len).rev() { + let state_index = self.0[index].index; + match states.history[state_index] { + TransactionState::TxPending => { + if state_index >= start_transaction_window { + previous_switch = Some(index); + } else { + if result.is_none() { + result = Some((index + 1, state_index)); + } + } + break; + }, + TransactionState::Pending => { + if state_index >= start_transaction_window { + previous_switch = Some(index); + } else { + if result.is_none() { + result = Some((index + 1, state_index)); + // continue iteration to find all `LayersEntry` from this + // start_transaction_window. + start_transaction_window = transaction_start_windows(states, state_index); + } else { + break; + } + } + }, + TransactionState::Dropped => (), + } + } + self.drop_last_values(result, previous_switch) + } + + /// Method to prune regarding a full state. + /// It also returns the last value as mutable. + /// Internally it acts like `get_mut` with an + /// additional cleaning capacity to clean committed + /// state if `prune_to_commit` is set to true. + pub(crate) fn get_mut_pruning( + &mut self, + states: &States, + prune_to_commit: bool, + ) -> Option> { + let self_len = self.0.len(); + if self_len == 0 { + return None; + } + let mut head_prune_index = 0; + debug_assert!(states.history.len() >= self_len); + let mut result = None; + let mut start_transaction_window = usize::max_value(); + let mut previous_switch = None; + for index in (0 .. self_len).rev() { + let state_index = self.0[index].index; + match states.history[state_index] { + state @ TransactionState::TxPending + | state @ TransactionState::Pending => { + if state_index >= start_transaction_window { + previous_switch = Some(index); + } else { + if result.is_none() { + result = Some((index + 1, state_index)); + if state == TransactionState::Pending { + start_transaction_window = transaction_start_windows( + states, + state_index, + ); + } + } else { + if !prune_to_commit { + break; + } + } + } + + if state_index < states.commit { + head_prune_index = index; + break; + } + }, + TransactionState::Dropped => (), + } + } + if prune_to_commit && head_prune_index > 0 && result.is_some() { + // Remove values before first value needed to read committed state. + self.remove_until(head_prune_index); + previous_switch = previous_switch.map(|switch| switch - head_prune_index); + result = result + .map(|(index, state_index)| (index - head_prune_index, state_index)); + } + self.drop_last_values(result, previous_switch) + } + + // Function used to remove all last values for `get_mut` and + // `get_mut_pruning`. + // + // It expects the `result` of the iteration that lookup + // for the latest non dropped value (coupled with its state + // index). That is to remove terminal dropped states. + // + // It also does remove values on committed state that are + // not needed (before another committed value). + // `previous_switch` is the index to the first unneeded value. + fn drop_last_values( + &mut self, + result: Option<(usize, usize)>, + previous_switch: Option, + ) -> Option> { + if let Some((index, state_index)) = result { + // Remove terminal values. + if index < self.0.len() { + self.0.truncate(index); + } + if let Some(switch_index) = previous_switch { + // We need to remove some value in between + // transaction start and this latest value. + if let Some(mut value) = self.0.pop() { + self.0.truncate(switch_index); + value.index = state_index; + self.push_unchecked(value); + } + Some((&mut self.0[switch_index].value, state_index).into()) + } else { + Some((&mut self.0[index - 1].value, state_index).into()) + } + } else { + self.0.clear(); + None + } + } + + /// Set a value, it uses a state history as parameter. + /// + /// This method uses `get_mut` and does remove pending + /// dropped value. + pub(crate) fn set(&mut self, states: &States, value: V) { + let last_state_index = states.num_states() - 1; + if let Some(v) = self.get_mut(states) { + if v.index == last_state_index { + *v.value = value; + return; + } + debug_assert!(v.index < last_state_index, "Layers expects \ + only new values at the latest state"); + } + self.push_unchecked(LayerEntry { + value, + index: last_state_index, + }); + } + + /// Extracts the committed value if there is one. + pub fn into_committed(mut self, states: &[TransactionState], committed: usize) -> Option { + let self_len = self.0.len(); + if self_len == 0 { + return None; + } + debug_assert!(states.len() >= self_len); + for index in (0 .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states, index) { + if h.index < committed { + self.0.truncate(index + 1); + return self.0.pop().map(|v| v.value); + } + } + } + None + } +} + +#[cfg(any(test, feature = "test-helpers"))] +impl Layers { + /// Get current number of stored historical values. + pub fn len(&self) -> usize { + self.0.len() + } +} + +#[cfg(test)] +impl Layers { + /// Create an history from a sequence of historical values. + pub fn from_iter(input: impl IntoIterator>) -> Self { + let mut history = Layers::default(); + for v in input { + history.push_unchecked(v); + } + history + } + + /// Get latest prospective value, excludes + /// committed values. + pub(crate) fn get_prospective(&self, states: &States) -> Option<&V> { + let self_len = self.0.len(); + if self_len == 0 { + return None; + } + debug_assert!(states.history.len() >= self_len); + for index in (states.commit .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { + return Some(h.value); + } + } + None + } + + /// Get latest committed value. + pub(crate) fn get_committed(&self, states: &States) -> Option<&V> { + let self_len = self.0.len(); + if self_len == 0 { + return None; + } + debug_assert!(states.history.len() >= self_len); + for index in (0 .. self_len).rev() { + if let Some(h) = self.get_pending_unchecked(states.history.as_ref(), index) { + if h.index < states.commit { + return Some(h.value); + } + } + } + None + } +} + +/// A default sample configuration to manage garbage collection +/// triggering. +/// +/// With this default values it should be very unlikelly that gc is needed +/// during a block processing for most use cases. +pub(crate) const DEFAULT_GC_CONF: GCConfiguration = GCConfiguration { + trigger_transaction_gc: 1_000_000, + trigger_commit_gc: 100_000, + add_content_size_unit: 64, +}; + +/// Garbage collection configuration. +/// It is designed to listen on two different event, transaction +/// or commit. +/// Transaction operations are `discard_transaction` and `commit_transaction`. +/// Prospective operations are `discard_prospective` and `commit_prospective`. +/// +/// Transaction can be call at any time, while prospective operation are only +/// call between extrinsics, therefore transaction treshold should be higher +/// than commit for being more frequent. +/// +/// It measures operation, that is write or delete storage, giving an idea +/// of a possible maximum size of used memory. +/// This is not a very good measurement and should be replace by +/// heap measurement or other metrics, at this point the gc even +/// if tested should not be required. +pub(crate) struct GCConfiguration { + /// Treshold in number of operation before running a garbage collection. + /// + /// Should be same as `TRIGGER_COMMIT_GC` or higher + /// (we most likely do not want lower as transaction are + /// possibly more frequent than commit). + pub trigger_transaction_gc: usize, + + /// Treshold in number of operation before running a garbage colletion + /// on a commit operation. + /// + /// We may want a lower value than for a transaction, even + /// a 1 if we want to do it between every operation. + pub trigger_commit_gc: usize, + + /// Used to count big content as multiple operations. + /// This is a number of octet. + /// Set to 0 to ignore. + pub add_content_size_unit: usize, +} + +impl GCConfiguration { + /// Cost depending on value if any. + pub fn operation_cost(&self, val: Option<&Vec>) -> usize { + let additional_cost = if self.add_content_size_unit > 0 { + if let Some(s) = val.as_ref() { + s.len() / self.add_content_size_unit + } else { + 0 + } + } else { 0 }; + 1 + additional_cost + } + +} + +/// Base code for fuzzing. +#[cfg(any(test, feature = "test-helpers"))] +pub mod fuzz { + use crate::overlayed_changes::OverlayedChanges; + use std::collections::HashMap; + + /// Size of key, max 255 + const KEY_SPACE: u8 = 20; + + /// Size of key, max 255 + const VALUE_SPACE: u8 = 50; + + /// Fuzz input against a stack of hash map reference implementation. + /// `check_compactness` add a check in the number of stored entry. + pub fn fuzz_transactions_inner(input: &[u8], check_compactness: bool) { + let mut input_index: usize = 0; + let mut overlayed = OverlayedChanges::default(); + let mut ref_overlayed = RefOverlayedChanges::default(); + + let mut actions = Vec::new(); + let mut values = Vec::new(); + loop { + let action: Actions = if let Some(v) = input.get(input_index) { + input_index += 1; + (*v).into() + } else { break }; + + actions.push(action); + match action { + Actions::CommitProspective => { + overlayed.commit_prospective(); + ref_overlayed.commit_prospective(); + }, + Actions::DropProspective => { + overlayed.discard_prospective(); + ref_overlayed.discard_prospective(); + }, + Actions::NewTransaction => { + overlayed.start_transaction(); + ref_overlayed.start_transaction(); + }, + Actions::CommitTransaction => { + overlayed.commit_transaction(); + ref_overlayed.commit_transaction(); + }, + Actions::DropTransaction => { + overlayed.discard_transaction(); + ref_overlayed.discard_transaction(); + }, + Actions::Insert => { + let key = if let Some(v) = input.get(input_index) { + input_index += 1; + v % KEY_SPACE + } else { break }; + let value = if let Some(v) = input.get(input_index) { + input_index += 1; + v % VALUE_SPACE + } else { break }; + values.push((key, value)); + overlayed.set_storage(vec![key], Some(vec![value])); + ref_overlayed.set_storage(vec![key], Some(vec![value])); + } + } + + } + + let mut success = true; + + let (check_value, len) = check_values(&overlayed, &ref_overlayed); + success &= check_value; + + if check_compactness { + let reference_size = ref_overlayed.total_length(); + overlayed.gc(true); + let size = overlayed.top_count_keyvalue_pair(); + if reference_size != size { + println!("inconsistent gc {} {}", size, reference_size); + success = false; + } + let (check_value, len_compactness) = check_values(&overlayed, &ref_overlayed); + success &= check_value; + success &= len_compactness == len; + } + ref_overlayed.commit_prospective(); + let ref_len = ref_overlayed.committed.len(); + if len != ref_len { + println!("inconsistent length {} {}", len, ref_len); + success = false; + } + if !success { + println!("fuzzing: \n {:x?}", (&actions, &values)); + println!("input: \n {:?}", &input); + } + + assert!(success); + } + + fn check_values( + overlayed: &OverlayedChanges, + ref_overlayed: &RefOverlayedChanges, + ) -> (bool, usize) { + let mut len = 0; + let mut success = true; + for (key, value) in overlayed.iter_values(None) { + let ref_value = ref_overlayed.storage(key); + if Some(value) != ref_value { + println!("at {:x?} different values {:x?} {:x?}", key, Some(value), ref_value); + success = false; + } + len += 1; + } + (success, len) + } + + #[derive(Clone, Copy, Debug)] + enum Actions { + Insert, + // Delete, same as an insert do not test. + CommitProspective, + DropProspective, + NewTransaction, + CommitTransaction, + DropTransaction, + } + + impl From for Actions { + fn from(v: u8) -> Self { + match (v as usize) * 100 / 255 { + v if v <= 5 => Actions::CommitProspective, + v if v <= 10 => Actions::DropProspective, + v if v <= 20 => Actions::NewTransaction, + v if v <= 30 => Actions::CommitTransaction, + v if v <= 40 => Actions::DropTransaction, + _ => Actions::Insert, + } + } + } + + /// A simple implementation of overlayed change + /// to use as a comparision. + /// It is partly incomplete (no child trie support, no change trie). + #[derive(Debug, Clone, Default)] + struct RefOverlayedChanges { + committed: HashMap, Vec>, + prospective: HashMap, Vec>, + transactions: Vec, Vec>>, + } + + impl RefOverlayedChanges { + fn discard_prospective(&mut self) { + self.transactions.clear(); + self.prospective.clear(); + } + + fn commit_prospective(&mut self) { + for _ in 0 .. self.transactions.len() { + self.commit_transaction(); + } + self.committed.extend(self.prospective.drain()); + } + + fn start_transaction(&mut self) { + self.transactions.push(Default::default()); + } + + fn discard_transaction(&mut self) { + if self.transactions.len() == 0 { + // clear prospective on no transaction started. + self.prospective.clear(); + } else { + let _ = self.transactions.pop(); + } + } + + /// Commit a transactional layer. + fn commit_transaction(&mut self) { + match self.transactions.len() { + 0 => (), + 1 => self.prospective.extend( + self.transactions.pop().expect("length just checked").into_iter() + ), + _ => { + let t = self.transactions.pop().expect("length just checked"); + self.transactions.last_mut().expect("length just checked") + .extend(t.into_iter()); + } + } + } + + fn set_storage(&mut self, key: Vec, val: Option>) { + if self.transactions.len() > 0 { + self.transactions.last_mut().expect("length just checked") + .insert(key, val.expect("fuzzer do not delete")); + } else { + self.prospective.insert(key, val.expect("fuzzer do not delete")); + } + } + + fn storage(&self, key: &[u8]) -> Option> { + for t in self.transactions.iter().rev() { + if let Some(v) = t.get(key) { + return Some(Some(v)); + } + } + if let Some(v) = self.prospective.get(key) { + return Some(Some(v)); + } + if let Some(v) = self.committed.get(key) { + return Some(Some(v)); + } + None + } + + fn total_length(&self) -> usize { + let tr_len: usize = self.transactions.iter() + .map(|l| l.len()).sum(); + self.committed.len() + self.prospective.len() + tr_len + } + } + + // Those are samples which found error during fuzzing. + // They are kept as a low cust rust test for non regression purpose. + #[test] + fn previous_fuzzed_error() { + let inputs = [ + vec![0x2,0xee,0xee,0x12,0x2,0x16,0x67,0xee,0xee,0xee,], + vec![50, 208, 50, 38, 46, 58, 209, 50, 216, 255, 255], + vec![0x98,0x98,0xf6,0x12,0xee,0x98,0xf9,], + vec![0xf1,0x0,0x0,0x1,0x38,0xb2,0x0,0x67,], + vec![238, 0, 36, 43, 50, 46, 38, 211, 0, 0, 61], + vec![50, 255, 38, 38, 186, 35, 46, 43, 46, 35, 255, 255, 102, 67], + vec![0x6e, 0xff, 0xf7, 0x0, 0x6e, 0xff, 0xff, 0x2d, 0xff, 0xff, 0xff, 0xe], + vec![0x2e,0x6e,0x22,0x32,0x2e,0x6e,0x22,0x32,0x3f,0x2e,], + vec![0xd9,0xff,0xfd,0x0,0xff,0xff,0xf8,0x1,0x92,0xff,0xbf,0x14,], + vec![0xef,0xdf,0xc1,0x0,0xc1,0xdf,0xc1,0x2b,0xf3,0xf3,0xb0,0x18, + 0xef,0xdf,0x2e,0x3a,0xef,0xdf,0x0,0xc1,0xf3,0x30,0x18,0xef,0xdf, + 0xc1,0x2b,0xf3,0xf3,0x30,0x17,0x0,0xdf,], + vec![0xff,0xd,0x0,0x61,0x8,0x9c,0xff,0x57,0xd3,0xff,0xd1,0xd1,0x26, + 0xff,0x33,0xff,0x24,0x1f,0xff,0xff,0xdd,0x0,0x8,0x7a,0x7f,0xff,0xff, + 0x26,0xff,0x7b,0xff,0xff,0x26,0xee,0xff,0xff,0x41,0x83], + ]; + for input in inputs.iter() { + fuzz_transactions_inner(&input[..], true); + } + } + + #[test] + fn test_internal_state() { + use super::{States, Layers, TransactionState}; + + let check_state = |global_state: &States, global_state_ref: &States, layers: &[&Layers], layers_ref: &[&[(usize, usize)]]| { + let layers_ref: Vec<_> = layers_ref.iter().map(|internal| + Layers::from_iter(internal.iter().map(Clone::clone).map(Into::into).into_iter()) + ).collect(); + let layers_ref_ref: Vec<_> = layers_ref.iter().collect(); + assert_eq!(global_state, global_state_ref); + assert_eq!(layers, &layers_ref_ref[..]); + }; + + let mut global_state = States::default(); + let ref_state = States::test_vector( + vec![TransactionState::Pending], + vec![0], + 0, + ); + let mut value1 = Layers::default(); + check_state(&global_state, &ref_state, &[&value1], &[&[]]); + value1.set(&global_state, 11); + check_state(&global_state, &ref_state, &[&value1], &[&[(11, 0)]]); + value1.set(&global_state, 12); + let mut value2 = Layers::default(); + check_state(&global_state, &ref_state, &[&value1, &value2], &[&[(12, 0)], &[]]); + global_state.start_transaction(); + let ref_state = States::test_vector( + vec![TransactionState::Pending, TransactionState::TxPending], + vec![0, 1], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], &[&[(12, 0)], &[]]); + value2.set(&global_state, 11); + let values_ref = &[ + &[(12, 0)][..], + &[(11, 1)][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + value1.set(&global_state, 13); + let values_ref = &[ + &[(12, 0), (13, 1)][..], + &[(11, 1)][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.start_transaction(); + let ref_state = States::test_vector( + vec![TransactionState::Pending, TransactionState::TxPending, TransactionState::TxPending], + vec![0, 1, 2], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.discard_transaction(); + let ref_state = States::test_vector( + vec![TransactionState::Pending, TransactionState::TxPending, TransactionState::Dropped, TransactionState::Pending], + vec![0, 1, 1, 1], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + value2.set(&global_state, 12); + let values_ref = &[ + &[(12, 0), (13, 1)][..], + &[(11, 1), (12, 3)][..], + ][..]; + assert_eq!(value1.get(global_state.as_ref()), (Some(&13), false)); + assert_eq!(value2.get(global_state.as_ref()), (Some(&12), false)); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.discard_transaction(); + let ref_state = States::test_vector( + vec![ + TransactionState::Pending, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Pending, + ], + vec![0, 0, 0, 0, 0], + 0, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + assert_eq!(value1.get(global_state.as_ref()), (Some(&12), true)); + assert_eq!(value2.get(global_state.as_ref()), (None, true)); + // update value 1 & value 2 by using get_mut. + assert!(value1.get_mut(&global_state).is_some()); + assert!(value2.get_mut(&global_state).is_none()); + assert_eq!(value1.get(global_state.as_ref()), (Some(&12), false)); + assert_eq!(value2.get(global_state.as_ref()), (None, false)); + let values_ref = &[ + &[(12, 0)][..], + &[][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.start_transaction(); + value2.set(&global_state, 14); + let ref_state = States::test_vector( + vec![ + TransactionState::Pending, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Pending, + TransactionState::TxPending, + ], + vec![0, 0, 0, 0, 0, 5], + 0, + ); + let values_ref = &[ + &[(12, 0)][..], + &[(14, 5)][..], + ][..]; + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + global_state.commit_prospective(); + let ref_state = States::test_vector( + vec![ + TransactionState::Pending, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Dropped, + TransactionState::Pending, + TransactionState::TxPending, + TransactionState::Pending, + ], + vec![6, 6, 6, 6, 6, 6, 6], + 6, + ); + check_state(&global_state, &ref_state, &[&value1, &value2], values_ref); + assert_eq!(value1.get(global_state.as_ref()), (Some(&12), false)); + assert_eq!(value2.get(global_state.as_ref()), (Some(&14), false)); + } +} diff --git a/test-utils/runtime/src/lib.rs b/test-utils/runtime/src/lib.rs index c0aea9a2ab599..752455e700470 100644 --- a/test-utils/runtime/src/lib.rs +++ b/test-utils/runtime/src/lib.rs @@ -283,8 +283,10 @@ cfg_if! { fn function_signature_changed() -> u64; fn fail_on_native() -> u64; fn fail_on_wasm() -> u64; - /// trie no_std testing + /// Trie no_std testing. fn use_trie() -> u64; + /// Transactional tests. + fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; fn benchmark_direct_call() -> u64; fn vec_with_capacity(size: u32) -> Vec; @@ -327,6 +329,8 @@ cfg_if! { fn fail_on_wasm() -> u64; /// trie no_std testing fn use_trie() -> u64; + /// Transactional tests. + fn use_transactions() -> u64; fn benchmark_indirect_call() -> u64; fn benchmark_direct_call() -> u64; fn vec_with_capacity(size: u32) -> Vec; @@ -579,6 +583,10 @@ cfg_if! { code_using_trie() } + fn use_transactions() -> u64 { + system::test_transactions() + } + fn benchmark_indirect_call() -> u64 { let function = benchmark_add_one; (0..1000).fold(0, |p, i| p + function(i)) @@ -773,6 +781,10 @@ cfg_if! { code_using_trie() } + fn use_transactions() -> u64 { + system::test_transactions() + } + fn benchmark_indirect_call() -> u64 { (0..10000).fold(0, |p, i| p + BENCHMARK_ADD_ONE.get()(i)) } @@ -1011,4 +1023,14 @@ mod tests { runtime_api.test_storage(&block_id).unwrap(); } + + #[test] + fn test_transactions() { + let client = TestClientBuilder::new() + .set_execution_strategy(ExecutionStrategy::AlwaysWasm) + .build(); + let runtime_api = client.runtime_api(); + let block_id = BlockId::Number(client.chain_info().best_number); + assert_eq!(runtime_api.use_transactions(&block_id).unwrap(), 1); + } } diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index c35850ae950e9..e9bfd0c76dae6 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -95,6 +95,35 @@ pub fn take_block_number() -> Option { Number::take() } +/// Return 1 if test successfull, 0 otherwise. +pub fn test_transactions() -> u64 { + let origin_value: Option = storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX); + let result: Result<(), _> = storage::with_transaction(|| { + if let Ok(val) = storage::with_transaction(|| { + storage::unhashed::put(well_known_keys::EXTRINSIC_INDEX, &99u32); + if storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX) == Some(99u32) { + Ok(99u32) + } else { + Err("Error setting value will revert") + } + }) { + if storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX) == Some(val) { + Err("expected revert for test") + } else { + Err("Unexpected revert") + } + } else { + Err("Unexpected revert") + } + }); + if result == Err("expected revert for test") { + if storage::unhashed::get(well_known_keys::EXTRINSIC_INDEX) == origin_value { + return 1; + } + } + 0 +} + #[derive(Copy, Clone)] enum Mode { Verify,