From 9736c50d88d16abbf1464516e9ea1965812a61c3 Mon Sep 17 00:00:00 2001 From: Robert Habermeier Date: Wed, 16 May 2018 16:56:24 +0200 Subject: [PATCH] Store trie nodes in DB (#157) * move responsibility of storage_root calculation to state backend * have `storage_root` produce a memoizable transaction * store trie nodes in kvdb * fix up test fallout * remove stray newline * Fix comment * test for setting and checking state data * fiddle with dependencies * all parity deps on same commit hash * fix network protocol registration --- Cargo.lock | 159 ++++++------- polkadot/api/src/lib.rs | 55 ++--- substrate/client/db/Cargo.toml | 4 + substrate/client/db/src/lib.rs | 301 +++++++++++++++++++++---- substrate/client/src/backend.rs | 9 +- substrate/client/src/block_builder.rs | 4 +- substrate/client/src/client.rs | 8 +- substrate/client/src/genesis.rs | 4 +- substrate/client/src/in_mem.rs | 4 +- substrate/network/src/service.rs | 4 +- substrate/state-machine/Cargo.toml | 3 - substrate/state-machine/src/backend.rs | 27 ++- substrate/state-machine/src/ext.rs | 55 +++-- substrate/state-machine/src/lib.rs | 26 +-- substrate/state-machine/src/testing.rs | 2 +- 15 files changed, 434 insertions(+), 231 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 733430322e59f..976b1ea5ee5d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -35,15 +35,6 @@ dependencies = [ "xdg 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "arrayvec" -version = "0.3.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)", - "odds 0.2.26 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "arrayvec" version = "0.4.7" @@ -104,8 +95,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustc_version 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -414,7 +403,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bigint 4.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", "plain_hasher 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", @@ -424,17 +412,12 @@ dependencies = [ [[package]] name = "ethcore-bytes" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" - -[[package]] -name = "ethcore-bytes" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" [[package]] name = "ethcore-crypto" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "quick-error 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -446,7 +429,7 @@ dependencies = [ [[package]] name = "ethcore-io" version = "1.12.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -459,26 +442,10 @@ dependencies = [ "timer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "ethcore-logger" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "arrayvec 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "isatty 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ethcore-logger" version = "1.12.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -494,7 +461,7 @@ dependencies = [ [[package]] name = "ethcore-network" version = "1.12.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-crypto 0.1.0 (git+https://github.com/paritytech/parity.git)", @@ -509,7 +476,7 @@ dependencies = [ [[package]] name = "ethcore-network-devp2p" version = "1.12.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", @@ -580,7 +547,7 @@ dependencies = [ [[package]] name = "ethkey" version = "0.3.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -703,10 +670,10 @@ dependencies = [ [[package]] name = "hashdb" version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ - "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -818,17 +785,6 @@ name = "ipnetwork" version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "isatty" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "itertools" version = "0.5.10" @@ -917,7 +873,7 @@ dependencies = [ [[package]] name = "keccak-hash" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -946,7 +902,7 @@ dependencies = [ [[package]] name = "kvdb" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -956,7 +912,7 @@ dependencies = [ [[package]] name = "kvdb-memorydb" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "kvdb 0.1.0 (git+https://github.com/paritytech/parity.git)", "parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -965,7 +921,7 @@ dependencies = [ [[package]] name = "kvdb-rocksdb" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1037,7 +993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "mem" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" [[package]] name = "memchr" @@ -1055,15 +1011,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "memorydb" version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ - "bigint 4.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hashdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.1.1 (git+https://github.com/paritytech/parity.git)", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rlp 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hash 0.1.0 (git+https://github.com/paritytech/parity.git)", + "plain_hasher 0.1.0 (git+https://github.com/paritytech/parity.git)", + "rlp 0.2.1 (git+https://github.com/paritytech/parity.git)", ] [[package]] @@ -1146,11 +1102,6 @@ dependencies = [ "libc 0.2.36 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "odds" -version = "0.2.26" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "ole32-sys" version = "0.2.0" @@ -1220,24 +1171,24 @@ dependencies = [ [[package]] name = "path" version = "0.1.0" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" [[package]] name = "patricia-trie" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ - "elastic-array 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "ethcore-logger 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hashdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethcore-bytes 0.1.0 (git+https://github.com/paritytech/parity.git)", + "ethcore-logger 1.12.0 (git+https://github.com/paritytech/parity.git)", + "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.1.1 (git+https://github.com/paritytech/parity.git)", + "keccak-hash 0.1.0 (git+https://github.com/paritytech/parity.git)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", - "memorydb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", - "rlp 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "triehash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rlp 0.2.1 (git+https://github.com/paritytech/parity.git)", + "triehash 0.1.0 (git+https://github.com/paritytech/parity.git)", ] [[package]] @@ -1245,6 +1196,15 @@ name = "percent-encoding" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "plain_hasher" +version = "0.1.0" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" +dependencies = [ + "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "plain_hasher" version = "0.1.0" @@ -1660,7 +1620,7 @@ dependencies = [ [[package]] name = "rlp" version = "0.2.1" -source = "git+https://github.com/paritytech/parity.git#1fa95ac236ab280afc9eccb4d6ea51b494f11422" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" dependencies = [ "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1936,11 +1896,15 @@ dependencies = [ name = "substrate-client-db" version = "0.1.0" dependencies = [ + "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hashdb 0.1.1 (git+https://github.com/paritytech/parity.git)", "kvdb 0.1.0 (git+https://github.com/paritytech/parity.git)", "kvdb-memorydb 0.1.0 (git+https://github.com/paritytech/parity.git)", "kvdb-rocksdb 0.1.0 (git+https://github.com/paritytech/parity.git)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)", "substrate-client 0.1.0", "substrate-codec 0.1.0", "substrate-primitives 0.1.0", @@ -2298,10 +2262,7 @@ name = "substrate-state-machine" version = "0.1.0" dependencies = [ "byteorder 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hashdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "hex-literal 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "memorydb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "patricia-trie 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-primitives 0.1.0", "triehash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2503,6 +2464,17 @@ dependencies = [ "smallvec 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "triehash" +version = "0.1.0" +source = "git+https://github.com/paritytech/parity.git#0ecbb3ec02a5b36c95f09a20d5958a374c32511f" +dependencies = [ + "elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ethereum-types 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "keccak-hash 0.1.0 (git+https://github.com/paritytech/parity.git)", + "rlp 0.2.1 (git+https://github.com/paritytech/parity.git)", +] + [[package]] name = "triehash" version = "0.1.0" @@ -2744,7 +2716,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" "checksum app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e73a24bad9bd6a94d6395382a6c69fe071708ae4409f763c5475e14ee896313d" -"checksum arrayvec 0.3.25 (registry+https://github.com/rust-lang/crates.io-index)" = "06f59fe10306bb78facd90d28c2038ad23ffaaefa85bac43c8a434cde383334f" "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" "checksum assert_matches 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9e772942dccdf11b368c31e044e4fca9189f80a773d2f0808379de65894cbf57" "checksum atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8352656fd42c30a0c3c89d26dea01e3b77c0ab2af18230835c15e2e13cd51859" @@ -2779,11 +2750,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum ethbloom 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a93a43ce2e9f09071449da36bfa7a1b20b950ee344b6904ff23de493b03b386" "checksum ethcore-bigint 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bcb5af77e74a8f70e9c3337e069c37bc82178ef1b459c02091f73c4ad5281eb5" "checksum ethcore-bytes 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" -"checksum ethcore-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3977c772cd6c5c22e1c7cfa208e4c3b746bd6c3a6c8eeec0999a6b2103015ad5" "checksum ethcore-crypto 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum ethcore-io 1.12.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum ethcore-logger 1.12.0 (git+https://github.com/paritytech/parity.git)" = "" -"checksum ethcore-logger 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1fd5813e49546030be7d134e775088d56b8ff4ab60617b90e93d4f0513da4c5b" "checksum ethcore-network 1.12.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum ethcore-network-devp2p 1.12.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum ethereum-types 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5cff74129deda8a155b729cad1a22dc3cdd08115abd1165079c519d0cab6917a" @@ -2803,7 +2772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" "checksum getopts 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "b900c08c1939860ce8b54dc6a89e26e00c04c380fd0e09796799bd7f12861e05" "checksum globset 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1e96ab92362c06811385ae9a34d2698e8a1160745e0c78fbb434a44c8de3fabc" -"checksum hashdb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d97be07c358c5b461268b4ce60304024c5fa5acfd4bd8cd743639f0252003cf5" +"checksum hashdb 0.1.1 (git+https://github.com/paritytech/parity.git)" = "" "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" "checksum hex 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "459d3cf58137bb02ad4adeef5036377ff59f066dbb82517b7192e3a5462a2abc" "checksum hex-literal 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bd546ef520ab3745f1aae5f2cdc6de9e6498e94d1ab138b9eb3ddfbf335847fb" @@ -2816,7 +2785,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum interleaved-ordered 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "141340095b15ed7491bd3d4ced9d20cebfb826174b6bb03386381f62b01e3d77" "checksum iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dbe6e417e7d0975db6512b90796e8ce223145ac4e33c377e4a42882a0e88bb08" "checksum ipnetwork 0.12.7 (registry+https://github.com/rust-lang/crates.io-index)" = "2134e210e2a024b5684f90e1556d5f71a1ce7f8b12e9ac9924c67fb36f63b336" -"checksum isatty 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8f2a233726c7bb76995cec749d59582e5664823b7245d4970354408f1d79a7a2" "checksum itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4833d6978da405305126af4ac88569b5d71ff758581ce5a987dbfa3755f694fc" "checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" "checksum jsonrpc-core 8.0.2 (git+https://github.com/paritytech/jsonrpc.git)" = "" @@ -2843,7 +2811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum mem 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d" "checksum memory_units 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "71d96e3f3c0b6325d8ccd83c33b28acb183edcb6c67938ba104ec546854b0882" -"checksum memorydb 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "013b7e4c5e10c764936ebc6bd3662d8e3c92292d267bf6a42ef3f5cad9c793ee" +"checksum memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)" = "" "checksum mime 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e2e00e17be181010a91dbfefb01660b17311059dc8c7f48b9017677721e732bd" "checksum mio 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "7da01a5e23070d92d99b1ecd1cd0af36447c6fd44b0fe283c2db199fa136724f" "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" @@ -2853,7 +2821,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum num-traits 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "9936036cc70fe4a8b2d338ab665900323290efb03983c86cbe235ae800ad8017" "checksum num-traits 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "dee092fcdf725aee04dd7da1d21debff559237d49ef1cb3e69bcb8ece44c7364" "checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" -"checksum odds 0.2.26 (registry+https://github.com/rust-lang/crates.io-index)" = "4eae0151b9dacf24fcc170d9995e511669a082856a91f958a2fe380bfab3fb22" "checksum ole32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2c49021782e5233cd243168edfa8037574afed4eba4bbaf538b3d8d1789d8c" "checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37" "checksum parity-wasm 0.27.4 (registry+https://github.com/rust-lang/crates.io-index)" = "1ba1ceaec13865445bcf05117867e4c6456d91c3617cdff2f3ef77b92b18cd12" @@ -2862,8 +2829,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum parking_lot 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3e7f7c9857874e54afeb950eebeae662b1e51a2493666d2ea4c0a5d91dcf0412" "checksum parking_lot_core 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "9f35048d735bb93dd115a0030498785971aab3234d311fbe273d020084d26bd8" "checksum path 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" -"checksum patricia-trie 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f1e2f638d79aba5c4a71a4f373df6e3cd702250a53b7f0ed4da1e2a7be9737ae" +"checksum patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum plain_hasher 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum plain_hasher 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "83ae80873992f511142c07d0ec6c44de5636628fdb7e204abd655932ea79d995" "checksum pretty_assertions 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28ea5118e2f41bfbc974b28d88c07621befd1fa5d6ec23549be96302a1a59dd2" "checksum proc-macro-hack 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ba8d4f9257b85eb6cdf13f055cea3190520aab1409ca2ab43493ea4820c25f0" @@ -2933,6 +2901,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tokio-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" "checksum tokio-timer 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6131e780037787ff1b3f8aad9da83bca02438b72277850dd6ad0d455e0e20efc" "checksum transaction-pool 1.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23303835df389f9c34ad45cacf392304193f974faaf48c30a4ece2b03da0ed57" +"checksum triehash 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum triehash 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9291c7f0fae44858b5e087dd462afb382354120003778f1695b44aab98c7abd7" "checksum twox-hash 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "475352206e7a290c5fccc27624a163e8d0d115f7bb60ca18a64fc9ce056d7435" "checksum uint 0.1.2 (git+https://github.com/rphmeier/primitives.git?branch=compile-for-wasm)" = "" diff --git a/polkadot/api/src/lib.rs b/polkadot/api/src/lib.rs index 731c19a49637d..666893585ddd8 100644 --- a/polkadot/api/src/lib.rs +++ b/polkadot/api/src/lib.rs @@ -168,10 +168,7 @@ macro_rules! with_runtime { $client.state_at(parent).map_err(Error::from).and_then(|state| { let mut changes = Default::default(); - let mut ext = state_machine::Ext { - overlay: &mut changes, - backend: &state, - }; + let mut ext = state_machine::Ext::new(&mut changes, &state); ::substrate_executor::with_native_environment(&mut ext, || { ::runtime::Executive::initialise_block(&header); @@ -278,51 +275,48 @@ pub struct ClientBlockBuilder { impl ClientBlockBuilder where S::Error: Into { - // executes a extrinsic, inherent or otherwise, without appending to the list + // initialises a block ready to allow extrinsics to be applied. fn initialise_block(&mut self) -> Result<()> { - let mut ext = state_machine::Ext { - overlay: &mut self.changes, - backend: &self.state, + let result = { + let mut ext = state_machine::Ext::new(&mut self.changes, &self.state); + let h = self.header.clone(); + + ::substrate_executor::with_native_environment( + &mut ext, + || runtime::Executive::initialise_block(&h), + ).map_err(Into::into) }; - let h = self.header.clone(); - - let result = ::substrate_executor::with_native_environment( - &mut ext, - || runtime::Executive::initialise_block(&h), - ).map_err(Into::into); - match result { Ok(_) => { - ext.overlay.commit_prospective(); + self.changes.commit_prospective(); Ok(()) } Err(e) => { - ext.overlay.discard_prospective(); + self.changes.discard_prospective(); Err(e) } } } - // executes a extrinsic, inherent or otherwise, without appending to the list + // executes a extrinsic, inherent or otherwise, without appending to the list. fn apply_extrinsic(&mut self, extrinsic: UncheckedExtrinsic) -> Result<()> { - let mut ext = state_machine::Ext { - overlay: &mut self.changes, - backend: &self.state, - }; + let result = { + let mut ext = state_machine::Ext::new(&mut self.changes, &self.state); - let result = ::substrate_executor::with_native_environment( - &mut ext, - move || runtime::Executive::apply_extrinsic(extrinsic), - ).map_err(Into::into); + ::substrate_executor::with_native_environment( + &mut ext, + move || runtime::Executive::apply_extrinsic(extrinsic), + ).map_err(Into::into) + }; match result { Ok(_) => { - ext.overlay.commit_prospective(); + self.changes.commit_prospective(); Ok(()) } Err(e) => { - ext.overlay.discard_prospective(); + self.changes.discard_prospective(); Err(e) } } @@ -344,10 +338,7 @@ impl BlockBuilder for ClientBlockBuilder } fn bake(mut self) -> Block { - let mut ext = state_machine::Ext { - overlay: &mut self.changes, - backend: &self.state, - }; + let mut ext = state_machine::Ext::new(&mut self.changes, &self.state); let final_header = ::substrate_executor::with_native_environment( &mut ext, diff --git a/substrate/client/db/Cargo.toml b/substrate/client/db/Cargo.toml index 2fae161dd51bd..2cf254aa6417b 100644 --- a/substrate/client/db/Cargo.toml +++ b/substrate/client/db/Cargo.toml @@ -8,6 +8,10 @@ parking_lot = "0.4" log = "0.3" kvdb = { git = "https://github.com/paritytech/parity.git" } kvdb-rocksdb = { git = "https://github.com/paritytech/parity.git" } +ethereum-types = "0.3" +hashdb = { git = "https://github.com/paritytech/parity.git" } +patricia-trie = { git = "https://github.com/paritytech/parity.git" } +memorydb = { git = "https://github.com/paritytech/parity.git" } substrate-primitives = { path = "../../../substrate/primitives" } substrate-client = { path = "../../../substrate/client" } substrate-state-machine = { path = "../../../substrate/state-machine" } diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 43136fc894bcb..9d2b875701404 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -17,9 +17,13 @@ //! Client backend that uses RocksDB database as storage. State is still kept in memory. extern crate substrate_client as client; +extern crate ethereum_types; extern crate kvdb_rocksdb; extern crate kvdb; +extern crate hashdb; +extern crate memorydb; extern crate parking_lot; +extern crate patricia_trie; extern crate substrate_state_machine as state_machine; extern crate substrate_primitives as primitives; extern crate substrate_runtime_support as runtime_support; @@ -34,17 +38,20 @@ extern crate kvdb_memorydb; use std::sync::Arc; use std::path::PathBuf; use std::collections::HashMap; -use parking_lot::RwLock; -use runtime_support::Hashable; -use primitives::blake2_256; + +use codec::Slicable; +use ethereum_types::H256 as TrieH256; +use hashdb::{DBValue, HashDB}; use kvdb_rocksdb::{Database, DatabaseConfig}; use kvdb::{KeyValueDB, DBTransaction}; +use memorydb::MemoryDB; +use parking_lot::RwLock; +use patricia_trie::{TrieDB, TrieDBMut, TrieError, Trie, TrieMut}; +use primitives::blake2_256; use primitives::block::{self, Id as BlockId, HeaderHash}; +use runtime_support::Hashable; use state_machine::backend::Backend as StateBackend; use state_machine::CodeExecutor; -use codec::Slicable; - -const STATE_HISTORY: block::Number = 64; /// Database settings. pub struct DatabaseSettings { @@ -176,7 +183,7 @@ impl BlockchainDb { }) } - fn read_db(&self, id: BlockId, column: Option) -> Result, client::error::Error> { + fn read_db(&self, id: BlockId, column: Option) -> Result, client::error::Error> { self.id(id).and_then(|key| match key { Some(key) => self.db.get(column, &key).map_err(db_err), @@ -272,38 +279,147 @@ impl client::backend::BlockImportOperation for BlockImportOperation { Ok(()) } - fn set_storage, Option>)>>(&mut self, changes: I) -> Result<(), client::error::Error> { - self.pending_state.commit(changes); + fn update_storage(&mut self, update: MemoryDB) -> Result<(), client::error::Error> { + self.pending_state.commit(update); Ok(()) } fn reset_storage, Vec)>>(&mut self, iter: I) -> Result<(), client::error::Error> { - self.pending_state.commit(iter.into_iter().map(|(k, v)| (k, Some(v)))); + // TODO: wipe out existing trie. + let (_, update) = self.pending_state.storage_root(iter.into_iter().map(|(k, v)| (k, Some(v)))); + self.pending_state.commit(update); Ok(()) } } +struct Ephemeral<'a> { + backing: &'a KeyValueDB, + overlay: &'a mut MemoryDB, +} + +impl<'a> HashDB for Ephemeral<'a> { + fn keys(&self) -> HashMap { + self.overlay.keys() // TODO: iterate backing + } + + fn get(&self, key: &TrieH256) -> Option { + match self.overlay.raw(key) { + Some((val, i)) => { + if i <= 0 { + None + } else { + Some(val) + } + } + None => { + match self.backing.get(::columns::STATE, &key.0[..]) { + Ok(x) => x, + Err(e) => { + warn!("Failed to read from DB: {}", e); + None + } + } + } + } + } + + fn contains(&self, key: &TrieH256) -> bool { + self.get(key).is_some() + } + + fn insert(&mut self, value: &[u8]) -> TrieH256 { + self.overlay.insert(value) + } + + fn emplace(&mut self, key: TrieH256, value: DBValue) { + self.overlay.emplace(key, value) + } + + fn remove(&mut self, key: &TrieH256) { + self.overlay.remove(key) + } +} + +/// DB-backed patricia trie state, transaction type is an overlay of changes to commit. pub struct DbState { - mem: state_machine::backend::InMemory, - changes: Vec<(Vec, Option>)>, + db: Arc, + root: TrieH256, + updates: MemoryDB, } impl state_machine::Backend for DbState { - type Error = state_machine::backend::Void; + type Error = client::error::Error; + type Transaction = MemoryDB; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { - self.mem.storage(key) + let mut read_overlay = MemoryDB::default(); + let eph = Ephemeral { + backing: &*self.db, + overlay: &mut read_overlay, + }; + + let map_e = |e: Box| ::client::error::Error::from(format!("Trie lookup error: {}", e)); + + TrieDB::new(&eph, &self.root).map_err(map_e)? + .get(key).map(|x| x.map(|val| val.to_vec())).map_err(map_e) } - fn commit(&mut self, changes: I) - where I: IntoIterator, Option>)> - { - self.changes = changes.into_iter().collect(); - self.mem.commit(self.changes.clone()); + fn commit(&mut self, transaction: MemoryDB) { + self.updates = transaction; } fn pairs(&self) -> Vec<(Vec, Vec)> { - self.mem.pairs() + let mut read_overlay = MemoryDB::default(); + let eph = Ephemeral { + backing: &*self.db, + overlay: &mut read_overlay, + }; + + let collect_all = || -> Result<_, Box> { + let trie = TrieDB::new(&eph, &self.root)?; + let mut v = Vec::new(); + for x in trie.iter()? { + let (key, value) = x?; + v.push((key.to_vec(), value.to_vec())); + } + + Ok(v) + }; + + match collect_all() { + Ok(v) => v, + Err(e) => { + debug!("Error extracting trie values: {}", e); + Vec::new() + } + } + } + + fn storage_root(&self, delta: I) -> ([u8; 32], MemoryDB) + where I: IntoIterator, Option>)> + { + let mut write_overlay = MemoryDB::default(); + let mut root = self.root; + { + let mut eph = Ephemeral { + backing: &*self.db, + overlay: &mut write_overlay, + }; + + let mut trie = TrieDBMut::from_existing(&mut eph, &mut root).expect("prior state root to exist"); // TODO: handle gracefully + for (key, change) in delta { + let result = match change { + Some(val) => trie.insert(&key, &val), + None => trie.remove(&key), // TODO: archive mode + }; + + if let Err(e) = result { + warn!("Failed to write to trie: {}", e); + } + } + } + + (root.0.into(), write_overlay) } } @@ -311,7 +427,6 @@ impl state_machine::Backend for DbState { pub struct Backend { db: Arc, blockchain: BlockchainDb, - old_states: RwLock>, } impl Backend { @@ -336,20 +451,9 @@ impl Backend { fn from_kvdb(db: Arc) -> Result { let blockchain = BlockchainDb::new(db.clone())?; - //load latest state - let mut state = state_machine::backend::InMemory::new(); - let mut old_states = HashMap::new(); - - { - let iter = db.iter(columns::STATE).map(|(k, v)| (k.to_vec(), Some(v.to_vec()))); - state.commit(iter); - old_states.insert(number_to_db_key(blockchain.meta.read().best_number), state); - } - Ok(Backend { db, blockchain, - old_states: RwLock::new(old_states) }) } } @@ -367,7 +471,7 @@ impl client::backend::Backend for Backend { }) } - fn commit_operation(&self, operation: Self::BlockImportOperation) -> Result<(), client::error::Error> { + fn commit_operation(&self, mut operation: Self::BlockImportOperation) -> Result<(), client::error::Error> { let mut transaction = DBTransaction::new(); if let Some(pending_block) = operation.pending_block { let hash: block::HeaderHash = pending_block.header.blake2_256().into(); @@ -384,17 +488,13 @@ impl client::backend::Backend for Backend { if pending_block.is_best { transaction.put(columns::META, meta::BEST_BLOCK, &key); } - for (key, val) in operation.pending_state.changes.into_iter() { - match val { - Some(v) => { transaction.put(columns::STATE, &key, &v); }, - None => { transaction.delete(columns::STATE, &key); }, + for (key, (val, rc)) in operation.pending_state.updates.drain() { + if rc > 0 { + transaction.put(columns::STATE, &key.0[..], &val); + } else { + transaction.delete(columns::STATE, &key.0[..]); } } - let mut states = self.old_states.write(); - states.insert(key, operation.pending_state.mem); - if number >= STATE_HISTORY { - states.remove(&number_to_db_key(number - STATE_HISTORY)); - } debug!("DB Commit {:?} ({})", hash, number); self.db.write(transaction).map_err(db_err)?; self.blockchain.update_meta(hash, number, pending_block.is_best); @@ -407,11 +507,31 @@ impl client::backend::Backend for Backend { } fn state_at(&self, block: BlockId) -> Result { - if let Some(state) = self.blockchain.id(block)?.and_then(|id| self.old_states.read().get(&id).cloned()) { - Ok(DbState { mem: state, changes: Vec::new() }) - } else { - Err(client::error::ErrorKind::UnknownBlock(block).into()) + use client::blockchain::Backend as BcBackend; + + // special case for genesis initialization + match block { + BlockId::Hash(h) if h == Default::default() => { + let mut root = TrieH256::default(); + let mut db = MemoryDB::default(); + TrieDBMut::new(&mut db, &mut root); + + return Ok(DbState { + db: self.db.clone(), + updates: Default::default(), + root, + }) + } + _ => {} } + + self.blockchain.header(block).and_then(|maybe_hdr| maybe_hdr.map(|hdr| { + DbState { + db: self.db.clone(), + updates: Default::default(), + root: hdr.state_root.0.into(), + } + }).ok_or_else(|| client::error::ErrorKind::UnknownBlock(block).into())) } } @@ -425,11 +545,17 @@ mod tests { #[test] fn block_hash_inserted_correctly() { let db = Backend::new_test(); - for i in 1..10 { + for i in 0..10 { assert!(db.blockchain().hash(i).unwrap().is_none()); { - let mut op = db.begin_operation(BlockId::Number(i - 1)).unwrap(); + let id = if i == 0 { + BlockId::Hash(Default::default()) + } else { + BlockId::Number(i - 1) + }; + + let mut op = db.begin_operation(id).unwrap(); let header = block::Header { number: i, parent_hash: Default::default(), @@ -444,11 +570,88 @@ mod tests { None, true, ).unwrap(); - op.set_storage(vec![].into_iter()).unwrap(); db.commit_operation(op).unwrap(); } assert!(db.blockchain().hash(i).unwrap().is_some()) } } + + #[test] + fn set_state_data() { + let db = Backend::new_test(); + { + let mut op = db.begin_operation(BlockId::Hash(Default::default())).unwrap(); + let mut header = block::Header { + number: 0, + parent_hash: Default::default(), + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let storage = vec![ + (vec![1, 3, 5], vec![2, 4, 6]), + (vec![1, 2, 3], vec![9, 9, 9]), + ]; + + header.state_root = op.pending_state.storage_root(storage + .iter() + .cloned() + .map(|(x, y)| (x, Some(y))) + ).0.into(); + + op.reset_storage(storage.iter().cloned()).unwrap(); + op.set_block_data( + header, + Some(vec![]), + None, + true + ).unwrap(); + + db.commit_operation(op).unwrap(); + + let state = db.state_at(BlockId::Number(0)).unwrap(); + + assert_eq!(state.storage(&[1, 3, 5]).unwrap(), Some(vec![2, 4, 6])); + assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); + assert_eq!(state.storage(&[5, 5, 5]).unwrap(), None); + + } + + { + let mut op = db.begin_operation(BlockId::Number(0)).unwrap(); + let mut header = block::Header { + number: 1, + parent_hash: Default::default(), + state_root: Default::default(), + digest: Default::default(), + extrinsics_root: Default::default(), + }; + + let storage = vec![ + (vec![1, 3, 5], None), + (vec![5, 5, 5], Some(vec![4, 5, 6])), + ]; + + let (root, overlay) = op.pending_state.storage_root(storage.iter().cloned()); + op.update_storage(overlay).unwrap(); + header.state_root = root.into(); + + op.set_block_data( + header, + Some(vec![]), + None, + true + ).unwrap(); + + db.commit_operation(op).unwrap(); + + let state = db.state_at(BlockId::Number(1)).unwrap(); + + assert_eq!(state.storage(&[1, 3, 5]).unwrap(), None); + assert_eq!(state.storage(&[1, 2, 3]).unwrap(), Some(vec![9, 9, 9])); + assert_eq!(state.storage(&[5, 5, 5]).unwrap(), Some(vec![4, 5, 6])); + } + } } diff --git a/substrate/client/src/backend.rs b/substrate/client/src/backend.rs index fdd5e39086743..1890668c26bae 100644 --- a/substrate/client/src/backend.rs +++ b/substrate/client/src/backend.rs @@ -16,7 +16,7 @@ //! Polkadot Client data backend -use state_machine; +use state_machine::backend::Backend as StateBackend; use error; use primitives::block::{self, Id as BlockId}; use primitives; @@ -24,14 +24,14 @@ use primitives; /// Block insertion operation. Keeps hold if the inserted block state and data. pub trait BlockImportOperation { /// Associated state backend type. - type State: state_machine::backend::Backend; + type State: StateBackend; /// Returns pending state. fn state(&self) -> error::Result<&Self::State>; /// Append block data to the transaction. fn set_block_data(&mut self, header: block::Header, body: Option, justification: Option, is_new_best: bool) -> error::Result<()>; /// Inject storage data into the database. - fn set_storage, Option>)>>(&mut self, changes: I) -> error::Result<()>; + fn update_storage(&mut self, update: ::Transaction) -> error::Result<()>; /// Inject storage data into the database replacing any existing data. fn reset_storage, Vec)>>(&mut self, iter: I) -> error::Result<()>; } @@ -43,9 +43,10 @@ pub trait Backend { /// Associated blockchain backend type. type Blockchain: ::blockchain::Backend; /// Associated state backend type. - type State: state_machine::backend::Backend; + type State: StateBackend; /// Begin a new block insertion transaction with given parent block id. + /// When constructing the genesis, this is called with all-zero hash. fn begin_operation(&self, block: BlockId) -> error::Result; /// Commit block insertion. fn commit_operation(&self, transaction: Self::BlockImportOperation) -> error::Result<()>; diff --git a/substrate/client/src/block_builder.rs b/substrate/client/src/block_builder.rs index 01dafa95a02e2..5190a01634677 100644 --- a/substrate/client/src/block_builder.rs +++ b/substrate/client/src/block_builder.rs @@ -69,7 +69,7 @@ impl BlockBuilder where /// can be validly executed (by executing it); if it is invalid, it'll be returned along with /// the error. Otherwise, it will return a mutable reference to self (in order to chain). pub fn push(&mut self, tx: Extrinsic) -> error::Result<()> { - let output = state_machine::execute(&self.state, &mut self.changes, &self.executor, "execute_transaction", + let (output, _) = state_machine::execute(&self.state, &mut self.changes, &self.executor, "execute_transaction", &vec![].and(&self.header).and(&tx))?; self.header = Header::decode(&mut &output[..]).expect("Header came straight out of runtime so must be valid"); self.transactions.push(tx); @@ -79,7 +79,7 @@ impl BlockBuilder where /// Consume the builder to return a valid `Block` containing all pushed transactions. pub fn bake(mut self) -> error::Result { self.header.extrinsics_root = ordered_trie_root(self.transactions.iter().map(Slicable::encode)).0.into(); - let output = state_machine::execute(&self.state, &mut self.changes, &self.executor, "finalise_block", + let (output, _) = state_machine::execute(&self.state, &mut self.changes, &self.executor, "finalise_block", &self.header.encode())?; self.header = Header::decode(&mut &output[..]).expect("Header came straight out of runtime so must be valid"); Ok(Block { diff --git a/substrate/client/src/client.rs b/substrate/client/src/client.rs index 967a24198d555..30ae5a261134a 100644 --- a/substrate/client/src/client.rs +++ b/substrate/client/src/client.rs @@ -229,7 +229,7 @@ impl Client where /// No changes are made. pub fn call(&self, id: &BlockId, method: &str, call_data: &[u8]) -> error::Result { let mut changes = OverlayedChanges::default(); - let return_data = state_machine::execute( + let (return_data, _) = state_machine::execute( &self.state_at(id)?, &mut changes, &self.executor, @@ -253,7 +253,7 @@ impl Client where overlay: &mut OverlayedChanges, f: F ) -> error::Result { - Ok(runtime_io::with_externalities(&mut Ext { backend: &self.state_at(id)?, overlay }, f)) + Ok(runtime_io::with_externalities(&mut Ext::new(overlay, &self.state_at(id)?), f)) } /// Create a new block, built on the head of the chain. @@ -299,7 +299,7 @@ impl Client where let mut transaction = self.backend.begin_operation(BlockId::Hash(header.parent_hash))?; let mut overlay = OverlayedChanges::default(); - state_machine::execute( + let (_out, storage_update) = state_machine::execute( transaction.state()?, &mut overlay, &self.executor, @@ -311,7 +311,7 @@ impl Client where let hash: block::HeaderHash = header.blake2_256().into(); trace!("Imported {}, (#{}), best={}, origin={:?}", hash, header.number, is_new_best, origin); transaction.set_block_data(header.clone(), body, Some(justification.uncheck().into()), is_new_best)?; - transaction.set_storage(overlay.drain())?; + transaction.update_storage(storage_update)?; self.backend.commit_operation(transaction)?; if origin == BlockOrigin::NetworkBroadcast || origin == BlockOrigin::Own || origin == BlockOrigin::ConsensusBroadcast { diff --git a/substrate/client/src/genesis.rs b/substrate/client/src/genesis.rs index c1cc82f90f3ed..c72f6addf7292 100644 --- a/substrate/client/src/genesis.rs +++ b/substrate/client/src/genesis.rs @@ -77,7 +77,7 @@ mod tests { let mut overlay = OverlayedChanges::default(); for tx in transactions.iter() { - let ret_data = execute( + let (ret_data, _) = execute( backend, &mut overlay, &Executor::new(), @@ -87,7 +87,7 @@ mod tests { header = Header::decode(&mut &ret_data[..]).unwrap(); } - let ret_data = execute( + let (ret_data, _) = execute( backend, &mut overlay, &Executor::new(), diff --git a/substrate/client/src/in_mem.rs b/substrate/client/src/in_mem.rs index b6a0341169be4..e3989b8ac4d8e 100644 --- a/substrate/client/src/in_mem.rs +++ b/substrate/client/src/in_mem.rs @@ -180,8 +180,8 @@ impl backend::BlockImportOperation for BlockImportOperation { Ok(()) } - fn set_storage, Option>)>>(&mut self, changes: I) -> error::Result<()> { - self.pending_state.commit(changes); + fn update_storage(&mut self, update: ::Transaction) -> error::Result<()> { + self.pending_state.commit(update); Ok(()) } diff --git a/substrate/network/src/service.rs b/substrate/network/src/service.rs index 377cbf8a92f4b..f32c88a26339b 100644 --- a/substrate/network/src/service.rs +++ b/substrate/network/src/service.rs @@ -35,6 +35,8 @@ use message::{Statement, LocalizedBftMessage}; /// Polkadot devp2p protocol id pub const DOT_PROTOCOL_ID: ProtocolId = *b"dot"; +const V0_PACKET_COUNT: u8 = 1; + /// Type that represents fetch completion future. pub type FetchFuture = oneshot::Receiver>; /// Type that represents statement stream. @@ -182,7 +184,7 @@ impl Service { Err(err) => warn!("Error starting network: {}", err), _ => {}, }; - self.network.register_protocol(self.handler.clone(), DOT_PROTOCOL_ID, 1, &[0u8]) + self.network.register_protocol(self.handler.clone(), DOT_PROTOCOL_ID, &[(0, V0_PACKET_COUNT)]) .unwrap_or_else(|e| warn!("Error registering polkadot protocol: {:?}", e)); } diff --git a/substrate/state-machine/Cargo.toml b/substrate/state-machine/Cargo.toml index 646c6d7f1d83c..647b9e40ddefb 100644 --- a/substrate/state-machine/Cargo.toml +++ b/substrate/state-machine/Cargo.toml @@ -6,9 +6,6 @@ description = "Substrate State Machine" [dependencies] substrate-primitives = { path = "../primitives", version = "0.1.0" } -hashdb = "0.1.1" -patricia-trie = "0.1.0" -memorydb = "0.1.1" triehash = "0.1" byteorder = "1.1" hex-literal = "0.1.0" diff --git a/substrate/state-machine/src/backend.rs b/substrate/state-machine/src/backend.rs index c906b74eeec48..9c90a47be161a 100644 --- a/substrate/state-machine/src/backend.rs +++ b/substrate/state-machine/src/backend.rs @@ -25,13 +25,20 @@ pub trait Backend { /// An error type when fetching data is not possible. type Error: super::Error; + /// Changes to be applied if committing + type Transaction; + /// Get keyed storage associated with specific address, or None if there is nothing associated. fn storage(&self, key: &[u8]) -> Result>, Self::Error>; - /// Commit updates to the backend and get new state. - fn commit(&mut self, changes: I) + /// Calculate the storage root, with given delta over what is already stored in + /// the backend, and produce a "transaction" that can be used to commit. + fn storage_root(&self, delta: I) -> ([u8; 32], Self::Transaction) where I: IntoIterator, Option>)>; + /// Commit updates to the backend and get new state. + fn commit(&mut self, tx: Self::Transaction); + /// Get all key/value pairs into a Vec. fn pairs(&self) -> Vec<(Vec, Vec)>; } @@ -57,14 +64,28 @@ pub type InMemory = HashMap, Vec>; impl Backend for InMemory { type Error = Void; + type Transaction = Vec<(Vec, Option>)>; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { Ok(self.get(key).map(Clone::clone)) } - fn commit(&mut self, changes: I) + fn storage_root(&self, delta: I) -> ([u8; 32], Self::Transaction) where I: IntoIterator, Option>)> { + let existing_pairs = self.iter().map(|(k, v)| (k.clone(), Some(v.clone()))); + + let transaction: Vec<_> = delta.into_iter().collect(); + let root = ::triehash::trie_root(existing_pairs.chain(transaction.iter().cloned()) + .collect::>() + .into_iter() + .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) + ).0; + + (root, transaction) + } + + fn commit(&mut self, changes: Self::Transaction) { for (key, val) in changes { match val { Some(v) => { self.insert(key, v); }, diff --git a/substrate/state-machine/src/ext.rs b/substrate/state-machine/src/ext.rs index 580e0209f4774..6ac27a71ef9e3 100644 --- a/substrate/state-machine/src/ext.rs +++ b/substrate/state-machine/src/ext.rs @@ -17,8 +17,6 @@ //! Conrete externalities implementation. use std::{error, fmt}; -use std::collections::HashMap; -use triehash::trie_root; use backend::Backend; use {Externalities, OverlayedChanges}; @@ -52,16 +50,37 @@ impl error::Error for Error { } /// Wraps a read-only backend, call executor, and current overlayed changes. -pub struct Ext<'a, B: 'a> { - /// The overlayed changes to write to. - pub overlay: &'a mut OverlayedChanges, - /// The storage backend to read from. - pub backend: &'a B, +pub struct Ext<'a, B: 'a + Backend> { + // The overlayed changes to write to. + overlay: &'a mut OverlayedChanges, + // The storage backend to read from. + backend: &'a B, + // The transaction necessary to commit to the backend. + transaction: Option<(B::Transaction, [u8; 32])>, +} + +impl<'a, B: 'a + Backend> Ext<'a, B> { + /// Create a new `Ext` from overlayed changes and read-only backend + pub fn new(overlay: &'a mut OverlayedChanges, backend: &'a B) -> Self { + Ext { + overlay, + backend, + transaction: None, + } + } + + /// Get the transaction necessary to update the backend. + pub fn transaction(mut self) -> B::Transaction { + let _ = self.storage_root(); + self.transaction.expect("transaction always set after calling storage root; qed").0 + } } #[cfg(test)] impl<'a, B: 'a + Backend> Ext<'a, B> { pub fn storage_pairs(&self) -> Vec<(Vec, Vec)> { + use std::collections::HashMap; + self.backend.pairs().iter() .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) .chain(self.overlay.committed.clone().into_iter()) @@ -82,6 +101,7 @@ impl<'a, B: 'a> Externalities for Ext<'a, B> } fn place_storage(&mut self, key: Vec, value: Option>) { + self.transaction = None; // wipe out the transaction since root will no longer be the same. self.overlay.set_storage(key, value); } @@ -89,13 +109,18 @@ impl<'a, B: 'a> Externalities for Ext<'a, B> 42 } - fn storage_root(&self) -> [u8; 32] { - trie_root(self.backend.pairs().into_iter() - .map(|(k, v)| (k, Some(v))) - .chain(self.overlay.committed.clone().into_iter()) - .chain(self.overlay.prospective.clone().into_iter()) - .collect::>() - .into_iter() - .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val)))).0 + fn storage_root(&mut self) -> [u8; 32] { + if let Some((_, ref root)) = self.transaction { + return root.clone(); + } + + // compute and memoize + let delta = self.overlay.committed.iter() + .chain(self.overlay.prospective.iter()) + .map(|(k, v)| (k.clone(), v.clone())); + + let (root, transaction) = self.backend.storage_root(delta); + self.transaction = Some((transaction, root)); + root } } diff --git a/substrate/state-machine/src/lib.rs b/substrate/state-machine/src/lib.rs index 4a8d377e2c5fe..72fa063dbb554 100644 --- a/substrate/state-machine/src/lib.rs +++ b/substrate/state-machine/src/lib.rs @@ -23,10 +23,6 @@ extern crate substrate_primitives as primitives; #[cfg_attr(test, macro_use)] extern crate hex_literal; -extern crate hashdb; -extern crate memorydb; - -extern crate patricia_trie; extern crate triehash; extern crate byteorder; @@ -130,7 +126,7 @@ pub trait Externalities { fn chain_id(&self) -> u64; /// Get the trie root of the current storage map. - fn storage_root(&self) -> [u8; 32]; + fn storage_root(&mut self) -> [u8; 32]; } /// Code execution engine. @@ -160,14 +156,11 @@ pub fn execute( exec: &Exec, method: &str, call_data: &[u8], -) -> Result, Box> +) -> Result<(Vec, B::Transaction), Box> { let result = { - let mut externalities = ext::Ext { - backend, - overlay: &mut *overlay - }; + let mut externalities = ext::Ext::new(overlay, backend); // make a copy. let code = externalities.storage(b":code") .ok_or(Box::new(ExecutionError::CodeEntryDoesNotExist) as Box)? @@ -178,13 +171,13 @@ pub fn execute( &code, method, call_data, - ) + ).map(move |out| (out, externalities.transaction())) }; match result { - Ok(out) => { + Ok(x) => { overlay.commit_prospective(); - Ok(out) + Ok(x) } Err(e) => { overlay.discard_prospective(); @@ -235,7 +228,7 @@ mod tests { #[test] fn overlayed_storage_root_works() { - let mut backend = InMemory::from(map![ + let backend = InMemory::from(map![ b"doe".to_vec() => b"reindeer".to_vec(), b"dog".to_vec() => b"puppyXXX".to_vec(), b"dogglesworth".to_vec() => b"catXXX".to_vec(), @@ -252,10 +245,7 @@ mod tests { b"doug".to_vec() => None ], }; - let ext = Ext { - backend: &mut backend, - overlay: &mut overlay, - }; + let mut ext = Ext::new(&mut overlay, &backend); const ROOT: [u8; 32] = hex!("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3"); assert_eq!(ext.storage_root(), ROOT); } diff --git a/substrate/state-machine/src/testing.rs b/substrate/state-machine/src/testing.rs index 9bd54c74f930d..e9b4a76db61b5 100644 --- a/substrate/state-machine/src/testing.rs +++ b/substrate/state-machine/src/testing.rs @@ -37,7 +37,7 @@ impl Externalities for TestExternalities { fn chain_id(&self) -> u64 { 42 } - fn storage_root(&self) -> [u8; 32] { + fn storage_root(&mut self) -> [u8; 32] { trie_root(self.clone()).0 } }