State machine call proof backend (#3945)

* drafting a proof extraction at the hashdb level (to include everything
for the 'call' proof case).

* use full proof by default (replace previous proof recorder).

* fix warnings.

* Cache value not found in proof recorder.

* Remove need or internal backend struct.

* fix type.

* doc

core/client/src/call_executor.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Substrate.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::{sync::Arc, cmp::Ord, panic::UnwindSafe, result, cell::RefCell, rc::Rc};
+use std::{sync::Arc, cmp::Ord, panic::UnwindSafe, result, cell::RefCell};
 use codec::{Encode, Decode};
 use sr_primitives::{
 	generic::BlockId, traits::Block as BlockT, traits::NumberFor,
@@ -81,7 +81,7 @@ where
 		execution_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
 		side_effects_handler: Option<OffchainExt>,
-		proof_recorder: &Option<Rc<RefCell<ProofRecorder<B>>>>,
+		proof_recorder: &Option<ProofRecorder<B>>,
 		enable_keystore: bool,
 	) -> error::Result<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone;
 
@@ -241,7 +241,7 @@ where
 		execution_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
 		side_effects_handler: Option<OffchainExt>,
-		recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
+		recorder: &Option<ProofRecorder<Block>>,
 		enable_keystore: bool,
 	) -> Result<NativeOrEncoded<R>, error::Error> where ExecutionManager<EM>: Clone {
 		match initialize_block {

core/client/src/client.rs

@@ -18,7 +18,7 @@
 
 use std::{
 	marker::PhantomData, collections::{HashSet, BTreeMap, HashMap}, sync::Arc,
-	panic::UnwindSafe, result, cell::RefCell, rc::Rc,
+	panic::UnwindSafe, result, cell::RefCell,
 };
 use log::{info, trace, warn};
 use futures03::channel::mpsc;
@@ -1494,7 +1494,7 @@ impl<B, E, Block, RA> CallRuntimeAt<Block> for Client<B, E, Block, RA> where
 		initialize_block: InitializeBlock<'a, Block>,
 		native_call: Option<NC>,
 		context: ExecutionContext,
-		recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
+		recorder: &Option<ProofRecorder<Block>>,
 	) -> error::Result<NativeOrEncoded<R>> {
 		let manager = match context {
 			ExecutionContext::BlockConstruction =>

core/client/src/light/call_executor.rs

@@ -16,7 +16,9 @@
 
 //! Methods that light client could use to execute runtime calls.
 
-use std::{sync::Arc, panic::UnwindSafe, result, cell::RefCell, rc::Rc};
+use std::{
+	sync::Arc, panic::UnwindSafe, result, cell::RefCell,
+};
 
 use codec::{Encode, Decode};
 use primitives::{
@@ -108,7 +110,7 @@ impl<Block, B, Local> CallExecutor<Block, Blake2Hasher> for
 		_manager: ExecutionManager<EM>,
 		native_call: Option<NC>,
 		side_effects_handler: Option<OffchainExt>,
-		recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
+		recorder: &Option<ProofRecorder<Block>>,
 		enable_keystore: bool,
 	) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
 		// there's no actual way/need to specify native/wasm execution strategy on light node
@@ -334,7 +336,7 @@ mod tests {
 			_execution_manager: ExecutionManager<EM>,
 			_native_call: Option<NC>,
 			_side_effects_handler: Option<OffchainExt>,
-			_proof_recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
+			_proof_recorder: &Option<ProofRecorder<Block>>,
 			_enable_keystore: bool,
 		) -> ClientResult<NativeOrEncoded<R>> where ExecutionManager<EM>: Clone {
 			unreachable!()

core/sr-api/proc-macro/src/decl_runtime_apis.rs

@@ -410,7 +410,7 @@ fn generate_call_api_at_calls(decl: &ItemTrait) -> Result<TokenStream> {
 				initialized_block: &std::cell::RefCell<Option<#crate_::BlockId<Block>>>,
 				native_call: Option<NC>,
 				context: #crate_::ExecutionContext,
-				recorder: &Option<std::rc::Rc<std::cell::RefCell<#crate_::ProofRecorder<Block>>>>,
+				recorder: &Option<#crate_::ProofRecorder<Block>>,
 			) -> std::result::Result<#crate_::NativeOrEncoded<R>, T::Error> {
 				let version = call_runtime_at.runtime_version_at(at)?;
 				use #crate_::InitializeBlock;

core/sr-api/proc-macro/src/impl_runtime_apis.rs

@@ -257,7 +257,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 			commit_on_success: std::cell::RefCell<bool>,
 			initialized_block: std::cell::RefCell<Option<#block_id>>,
 			changes: std::cell::RefCell<#crate_::OverlayedChanges>,
-			recorder: Option<std::rc::Rc<std::cell::RefCell<#crate_::ProofRecorder<#block>>>>,
+			recorder: Option<#crate_::ProofRecorder<#block>>,
 		}
 
 		// `RuntimeApi` itself is not threadsafe. However, an instance is only available in a
@@ -300,11 +300,9 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 				self.recorder
 					.take()
 					.map(|recorder| {
-						let trie_nodes = recorder
-							.borrow_mut()
-							.drain()
-							.into_iter()
-							.map(|record| record.data)
+						let trie_nodes = recorder.read()
+							.iter()
+							.filter_map(|(_k, v)| v.as_ref().map(|v| v.to_vec()))
 							.collect();
 						#crate_::StorageProof::new(trie_nodes)
 					})
@@ -339,7 +337,7 @@ fn generate_runtime_api_base_structures(impls: &[ItemImpl]) -> Result<TokenStrea
 					&Self,
 					&std::cell::RefCell<#crate_::OverlayedChanges>,
 					&std::cell::RefCell<Option<#crate_::BlockId<#block>>>,
-					&Option<std::rc::Rc<std::cell::RefCell<#crate_::ProofRecorder<#block>>>>,
+					&Option<#crate_::ProofRecorder<#block>>,
 				) -> std::result::Result<#crate_::NativeOrEncoded<R>, E>,
 				E,
 			>(

core/sr-api/src/lib.rs

@@ -59,16 +59,14 @@ use rstd::result;
 pub use codec::{Encode, Decode};
 use primitives::OpaqueMetadata;
 #[cfg(feature = "std")]
-use std::{panic::UnwindSafe, cell::RefCell, rc::Rc};
-#[cfg(feature = "std")]
-use primitives::Hasher as HasherT;
+use std::{panic::UnwindSafe, cell::RefCell};
 
 pub use sr_api_proc_macro::{decl_runtime_apis, impl_runtime_apis};
 
 #[cfg(feature = "std")]
 /// A type that records all accessed trie nodes and generates a proof out of it.
 pub type ProofRecorder<B> = state_machine::ProofRecorder<
-	<<<<B as BlockT>::Header as HeaderT>::Hashing as HashT>::Hasher as HasherT>::Out
+	<<<B as BlockT>::Header as HeaderT>::Hashing as HashT>::Hasher
 >;
 
 /// Something that can be constructed to a runtime api.
@@ -168,7 +166,7 @@ pub trait CallRuntimeAt<Block: BlockT> {
 		initialize_block: InitializeBlock<'a, Block>,
 		native_call: Option<NC>,
 		context: ExecutionContext,
-		recorder: &Option<Rc<RefCell<ProofRecorder<Block>>>>,
+		recorder: &Option<ProofRecorder<Block>>,
 	) -> Result<NativeOrEncoded<R>, Self::Error>;
 
 	/// Returns the runtime version at the given block.

core/state-machine/src/changes_trie/changes_iterator.rs

@@ -28,7 +28,7 @@ use crate::changes_trie::input::{DigestIndex, ExtrinsicIndex, DigestIndexValue,
 use crate::changes_trie::storage::{TrieBackendAdapter, InMemoryStorage};
 use crate::changes_trie::input::ChildIndex;
 use crate::changes_trie::surface_iterator::{surface_iterator, SurfaceIterator};
-use crate::proving_backend::ProvingBackendEssence;
+use crate::proving_backend::ProvingBackendRecorder;
 use crate::trie_backend_essence::{TrieBackendEssence};
 
 /// Return changes of given key at given blocks range.
@@ -366,7 +366,7 @@ impl<'a, H, Number> Iterator for ProvingDrilldownIterator<'a, H, Number>
 		let proof_recorder = &mut *self.proof_recorder.try_borrow_mut()
 			.expect("only fails when already borrowed; storage() is non-reentrant; qed");
 		self.essence.next(|storage, root, key|
-			ProvingBackendEssence::<_, H> {
+			ProvingBackendRecorder::<_, H> {
 				backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
 				proof_recorder,
 			}.storage(key))

core/state-machine/src/changes_trie/prune.rs

@@ -20,7 +20,7 @@ use hash_db::Hasher;
 use trie::Recorder;
 use log::warn;
 use num_traits::{One, Zero};
-use crate::proving_backend::ProvingBackendEssence;
+use crate::proving_backend::ProvingBackendRecorder;
 use crate::trie_backend_essence::TrieBackendEssence;
 use crate::changes_trie::{AnchorBlockId, Configuration, Storage, BlockNumber};
 use crate::changes_trie::storage::TrieBackendAdapter;
@@ -122,7 +122,7 @@ fn prune_trie<S: Storage<H, Number>, H: Hasher, Number: BlockNumber, F: FnMut(H:
 	// (effectively - all changes trie nodes)
 	let mut proof_recorder: Recorder<H::Out> = Default::default();
 	{
-		let mut trie = ProvingBackendEssence::<_, H> {
+		let mut trie = ProvingBackendRecorder::<_, H> {
 			backend: &TrieBackendEssence::new(TrieBackendAdapter::new(storage), root),
 			proof_recorder: &mut proof_recorder,
 		};

core/state-machine/src/lib.rs

@@ -60,7 +60,7 @@ pub use changes_trie::{
 pub use overlayed_changes::OverlayedChanges;
 pub use proving_backend::{
 	create_proof_check_backend, create_proof_check_backend_storage, merge_storage_proofs,
-	Recorder as ProofRecorder, ProvingBackend, StorageProof,
+	ProofRecorder, ProvingBackend, ProvingBackendRecorder, StorageProof,
 };
 pub use trie_backend_essence::{TrieBackendStorage, Storage};
 pub use trie_backend::TrieBackend;