Bump Frontier and Friends Pre v0.5 (paritytech#176)

* Update substrate -> c9d9365 (master)

* Update frontier -> 7e6f52e (v0.5-hotfixes)

* Update Cumulus and re-resolve some dependency versions

* Use same polkadot commit as Cumulus

* Resolve dependencies related to primitive-types

* Add SS58Prefix chain id to runtime config

* Implement query_fee_details from TransactionPayment trait

* Ensure frontier always points to v0.5-hotfixes

* Let cargo update Cargo.lock

* Move pubsub and rpc "hotfixes" back to frontier

* Update cumulus and polkadot

* correct hotfixes branch

* Made it to compile errors in actual moonbeam code :)

* runtime compiles

* new logger structure

* Fix call to start_collator

* Totally not-dubious changes to Cargo.lock

* Impl new query_fee_details fn

* Revert "Impl new query_fee_details fn"

This reverts commit 8c943f7279a9a6a3f453eb9e16beba3c501bd62b.

* Update rpc and standalone to use txn graph

* Cargo.lock changes again

* Add graph to FullDeps @ parachain service

* Run cargo fmt

* Appease editorconfig

* add web3api server

* Spawn background task for pending transactions

* Add log subscription tests

* Add pending transaction RPC test

* Prefer spaces over tabs in typescript tests

* s/describeWithFrontier/describeWithMoonbeam/

* Post-merge clean up

* prettier

* rustfmt

* fix test formatting

* Fixes tests for pallet stake

* Update runtime/Cargo.toml

Co-authored-by: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>

* reapply de-alias

* restore missing cargo dep

* Fixes new tests

* Fixes some tests

* Fix test-pending-pool

* Fix test-subscription

* Update Cargo.lock

* Improves polkadot api test (WIP)

* Better polkadot tests

Co-authored-by: Joshy Orndorff <admin@joshyorndorff.com>
Co-authored-by: crystalin <alan.sapede@gmail.com>
Co-authored-by: Joshy Orndorff <JoshOrndorff@users.noreply.github.com>
Co-authored-by: Crystalin <alan@purestake.com>
Co-authored-by: tgmichel <telmo@purestake.com>

node/parachain/Cargo.toml

@@ -62,9 +62,13 @@ pallet-transaction-payment-rpc = { git = "https://github.com/paritytech/substrat
 sp-block-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-consensus-manual-seal = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-fc-rpc = { git = "https://github.com/purestake/frontier", branch = "v0.4-hotfixes" }
-fp-rpc = { git = "https://github.com/purestake/frontier", branch = "v0.4-hotfixes" }
-fc-consensus = { git = "https://github.com/purestake/frontier", branch = "v0.4-hotfixes" }
+evm = { package = "pallet-evm", git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
+ethereum = { package = "pallet-ethereum", git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
+
+fc-consensus = { git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
+fp-consensus = { git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
+fc-rpc-core = { git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
+fc-rpc = { git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
 
 author-inherent = { path = "../../pallets/author-inherent"}
 

node/parachain/src/command.rs

@@ -24,8 +24,8 @@ use moonbeam_runtime::Block;
 use parity_scale_codec::Encode;
 use polkadot_parachain::primitives::AccountIdConversion;
 use sc_cli::{
-	ChainSpec, CliConfiguration, DefaultConfigurationValues, ImportParams, KeystoreParams,
-	NetworkParams, Result, RuntimeVersion, SharedParams, SubstrateCli,
+	ChainSpec, CliConfiguration, DefaultConfigurationValues, ImportParams, InitLoggerParams,
+	KeystoreParams, NetworkParams, Result, RuntimeVersion, SharedParams, SubstrateCli,
 };
 use sc_service::{
 	config::{BasePath, PrometheusConfig},
@@ -206,7 +206,10 @@ pub fn run() -> Result<()> {
 			})
 		}
 		Some(Subcommand::ExportGenesisState(params)) => {
-			sc_cli::init_logger("", sc_tracing::TracingReceiver::Log, None, false)?;
+			sc_cli::init_logger(InitLoggerParams {
+				tracing_receiver: sc_tracing::TracingReceiver::Log,
+				..Default::default()
+			})?;
 
 			let block: Block = generate_genesis_block(&load_spec(
 				&params.chain.clone().unwrap_or_default(),
@@ -228,7 +231,10 @@ pub fn run() -> Result<()> {
 			Ok(())
 		}
 		Some(Subcommand::ExportGenesisWasm(params)) => {
-			sc_cli::init_logger("", sc_tracing::TracingReceiver::Log, None, false)?;
+			sc_cli::init_logger(InitLoggerParams {
+				tracing_receiver: sc_tracing::TracingReceiver::Log,
+				..Default::default()
+			})?;
 
 			let raw_wasm_blob =
 				extract_genesis_wasm(&cli.load_spec(&params.chain.clone().unwrap_or_default())?)?;

node/parachain/src/service.rs

@@ -19,17 +19,22 @@ use cumulus_service::{
 	prepare_node_config, start_collator, start_full_node, StartCollatorParams, StartFullNodeParams,
 };
 use fc_consensus::FrontierBlockImport;
+use fc_rpc_core::types::PendingTransactions;
 use moonbeam_runtime::{opaque::Block, RuntimeApi};
 use parity_scale_codec::Encode;
 use polkadot_primitives::v0::CollatorPair;
+use sc_client_api::BlockchainEvents;
 use sc_executor::native_executor_instance;
 pub use sc_executor::NativeExecutor;
 use sc_service::{Configuration, PartialComponents, Role, TFullBackend, TFullClient, TaskManager};
 use sp_core::{Pair, H160};
 use sp_inherents::InherentDataProviders;
 use sp_runtime::traits::BlakeTwo256;
 use sp_trie::PrefixedMemoryDB;
-use std::sync::Arc;
+use std::{
+	collections::HashMap,
+	sync::{Arc, Mutex},
+};
 // Our native executor instance.
 native_executor_instance!(
 	pub Executor,
@@ -75,7 +80,10 @@ pub fn new_partial(
 		(),
 		sp_consensus::import_queue::BasicQueue<Block, PrefixedMemoryDB<BlakeTwo256>>,
 		sc_transaction_pool::FullPool<Block, FullClient>,
-		FrontierBlockImport<Block, Arc<FullClient>, FullClient>,
+		(
+			FrontierBlockImport<Block, Arc<FullClient>, FullClient>,
+			PendingTransactions,
+		),
 	>,
 	sc_service::Error,
 > {
@@ -94,6 +102,8 @@ pub fn new_partial(
 		client.clone(),
 	);
 
+	let pending_transactions: PendingTransactions = Some(Arc::new(Mutex::new(HashMap::new())));
+
 	let frontier_block_import = FrontierBlockImport::new(client.clone(), client.clone(), true);
 
 	let import_queue = cumulus_consensus::import_queue::import_queue(
@@ -113,7 +123,7 @@ pub fn new_partial(
 		transaction_pool,
 		inherent_data_providers,
 		select_chain: (),
-		other: frontier_block_import,
+		other: (frontier_block_import, pending_transactions),
 	};
 
 	Ok(params)
@@ -167,7 +177,7 @@ where
 	let transaction_pool = params.transaction_pool.clone();
 	let mut task_manager = params.task_manager;
 	let import_queue = params.import_queue;
-	let block_import = params.other;
+	let (block_import, pending_transactions) = params.other;
 	let (network, network_status_sinks, system_rpc_tx, start_network) =
 		sc_service::build_network(sc_service::BuildNetworkParams {
 			config: &parachain_config,
@@ -187,14 +197,16 @@ where
 		let client = client.clone();
 		let pool = transaction_pool.clone();
 		let network = network.clone();
-
+		let pending = pending_transactions.clone();
 		Box::new(move |deny_unsafe, _| {
 			let deps = moonbeam_rpc::FullDeps {
 				client: client.clone(),
 				pool: pool.clone(),
+				graph: pool.pool().clone(),
 				deny_unsafe,
 				is_authority,
 				network: network.clone(),
+				pending_transactions: pending.clone(),
 				command_sink: None,
 			};
 
@@ -218,6 +230,53 @@ where
 		system_rpc_tx,
 	})?;
 
+	// Spawn Frontier pending transactions maintenance task (as essential, otherwise we leak).
+	if pending_transactions.is_some() {
+		use fp_consensus::{ConsensusLog, FRONTIER_ENGINE_ID};
+		use futures::StreamExt;
+		use sp_runtime::generic::OpaqueDigestItemId;
+
+		const TRANSACTION_RETAIN_THRESHOLD: u64 = 5;
+		task_manager.spawn_essential_handle().spawn(
+			"frontier-pending-transactions",
+			client
+				.import_notification_stream()
+				.for_each(move |notification| {
+					if let Ok(locked) = &mut pending_transactions.clone().unwrap().lock() {
+						// As pending transactions have a finite lifespan anyway
+						// we can ignore MultiplePostRuntimeLogs error checks.
+						let mut frontier_log: Option<_> = None;
+						for log in notification.header.digest.logs {
+							let log = log.try_to::<ConsensusLog>(OpaqueDigestItemId::Consensus(
+								&FRONTIER_ENGINE_ID,
+							));
+							if let Some(log) = log {
+								frontier_log = Some(log);
+							}
+						}
+
+						let imported_number: u64 = notification.header.number as u64;
+
+						if let Some(ConsensusLog::EndBlock {
+							block_hash: _,
+							transaction_hashes,
+						}) = frontier_log
+						{
+							// Retain all pending transactions that were not
+							// processed in the current block.
+							locked.retain(|&k, _| !transaction_hashes.contains(&k));
+						}
+						locked.retain(|_, v| {
+							// Drop all the transactions that exceeded the given lifespan.
+							let lifespan_limit = v.at_block + TRANSACTION_RETAIN_THRESHOLD;
+							lifespan_limit > imported_number
+						});
+					}
+					futures::future::ready(())
+				}),
+		);
+	}
+
 	let announce_block = {
 		let network = network.clone();
 		Arc::new(move |hash, data| network.announce_block(hash, data))

node/rpc/Cargo.toml

@@ -29,17 +29,9 @@ substrate-frame-rpc-system = { git = "https://github.com/paritytech/substrate",
 pallet-transaction-payment-rpc = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sp-block-builder = { git = "https://github.com/paritytech/substrate", branch = "master" }
 sc-consensus-manual-seal = { git = "https://github.com/paritytech/substrate", branch = "master" }
+sc-transaction-graph = { git = "https://github.com/paritytech/substrate", branch = "master" }
 
-fc-rpc = { git = "https://github.com/purestake/frontier", branch = "v0.4-hotfixes" }
-fp-rpc = { git = "https://github.com/purestake/frontier", branch = "v0.4-hotfixes" }
+fc-rpc-core = { git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
+fc-rpc = { git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
+fp-rpc = { git = "https://github.com/purestake/frontier", branch = "v0.5-hotfixes" }
 
-# Dependencies I've added for the server hotfixes
-ethereum = { version = "0.5", features = ["with-codec"] }
-ethereum-types = "0.9.0"
-sha3 = "0.8"
-sp-io = { git = "https://github.com/paritytech/substrate.git", branch = "master" }
-sp-storage = { git = "https://github.com/paritytech/substrate.git", branch = "master" }
-fc-rpc-core = { git = "https://github.com/purestake/frontier", branch = "v0.4-hotfixes" }
-rlp = "0.4"
-fc-consensus = { git = "https://github.com/purestake/frontier", branch = "v0.4-hotfixes" }
-log = "0.4.8"

node/rpc/src/lib.rs

@@ -16,13 +16,9 @@
 
 //! A collection of node-specific RPC methods.
 
-// Our drop-in replacements for Frontier's RPC servers.
-mod pubsub_hotfixes;
-mod server_hotfixes;
-
 use std::{fmt, sync::Arc};
 
-use fc_rpc::HexEncodedIdProvider;
+use fc_rpc_core::types::PendingTransactions;
 use jsonrpc_pubsub::manager::SubscriptionManager;
 use moonbeam_runtime::{opaque::Block, AccountId, Balance, Hash, Index};
 use sc_client_api::{
@@ -33,6 +29,7 @@ use sc_consensus_manual_seal::rpc::{EngineCommand, ManualSeal, ManualSealApi};
 use sc_network::NetworkService;
 use sc_rpc::SubscriptionTaskExecutor;
 use sc_rpc_api::DenyUnsafe;
+use sc_transaction_graph::{ChainApi, Pool};
 use sp_api::ProvideRuntimeApi;
 use sp_block_builder::BlockBuilder;
 use sp_blockchain::{Error as BlockChainError, HeaderBackend, HeaderMetadata};
@@ -52,24 +49,28 @@ pub struct LightDeps<C, F, P> {
 }
 
 /// Full client dependencies.
-pub struct FullDeps<C, P> {
+pub struct FullDeps<C, P, A: ChainApi> {
 	/// The client instance to use.
 	pub client: Arc<C>,
 	/// Transaction pool instance.
 	pub pool: Arc<P>,
+	/// Graph pool instance.
+	pub graph: Arc<Pool<A>>,
 	/// Whether to deny unsafe calls
 	pub deny_unsafe: DenyUnsafe,
 	/// The Node authority flag
 	pub is_authority: bool,
 	/// Network service
 	pub network: Arc<NetworkService<Block, Hash>>,
+	/// Ethereum pending transactions.
+	pub pending_transactions: PendingTransactions,
 	/// Manual seal command sink
 	pub command_sink: Option<futures::channel::mpsc::Sender<EngineCommand<Hash>>>,
 }
 
 /// Instantiate all Full RPC extensions.
-pub fn create_full<C, P, BE>(
-	deps: FullDeps<C, P>,
+pub fn create_full<C, P, BE, A>(
+	deps: FullDeps<C, P, A>,
 	subscription_task_executor: SubscriptionTaskExecutor,
 ) -> jsonrpc_core::IoHandler<sc_rpc::Metadata>
 where
@@ -82,25 +83,27 @@ where
 	C::Api: substrate_frame_rpc_system::AccountNonceApi<Block, AccountId, Index>,
 	C::Api: BlockBuilder<Block>,
 	C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi<Block, Balance>,
+	A: ChainApi<Block = Block> + 'static,
 	C::Api: fp_rpc::EthereumRuntimeRPCApi<Block>,
 	<C::Api as sp_api::ApiErrorExt>::Error: fmt::Debug,
 	P: TransactionPool<Block = Block> + 'static,
 {
-	use fc_rpc::{NetApi, NetApiServer, Web3Api, Web3ApiServer};
+	use fc_rpc::{
+		EthApi, EthApiServer, EthPubSubApi, EthPubSubApiServer, HexEncodedIdProvider, NetApi,
+		NetApiServer, Web3Api, Web3ApiServer,
+	};
 	use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApi};
 	use substrate_frame_rpc_system::{FullSystem, SystemApi};
-	// Our drop in replacements for the Eth APIs. These can be removed after
-	// https://github.com/paritytech/frontier/pull/199 lands
-	use pubsub_hotfixes::{EthPubSubApi, EthPubSubApiServer};
-	use server_hotfixes::{EthApi, EthApiServer};
 
 	let mut io = jsonrpc_core::IoHandler::default();
 	let FullDeps {
 		client,
 		pool,
+		graph,
 		deny_unsafe,
 		is_authority,
 		network,
+		pending_transactions,
 		command_sink,
 	} = deps;
 
@@ -113,14 +116,16 @@ where
 		client.clone(),
 	)));
 
-	// We currently don't want to support signing in the node. Users should prefer external tools
-	// for transaction signing. So just pass in an empty vector of signers.
+	// TODO: are we supporting signing?
 	let signers = Vec::new();
+
 	io.extend_with(EthApiServer::to_delegate(EthApi::new(
 		client.clone(),
 		pool.clone(),
+		graph,
 		moonbeam_runtime::TransactionConverter,
 		network.clone(),
+		pending_transactions.clone(),
 		signers,
 		is_authority,
 	)));
@@ -130,21 +135,24 @@ where
 	)));
 	io.extend_with(Web3ApiServer::to_delegate(Web3Api::new(client.clone())));
 	io.extend_with(EthPubSubApiServer::to_delegate(EthPubSubApi::new(
-		pool,
-		client,
-		network,
+		pool.clone(),
+		client.clone(),
+		network.clone(),
 		SubscriptionManager::<HexEncodedIdProvider>::with_id_provider(
 			HexEncodedIdProvider::default(),
 			Arc::new(subscription_task_executor),
 		),
 	)));
 
-	if let Some(command_sink) = command_sink {
-		io.extend_with(
-			// We provide the rpc handler with the sending end of the channel to allow the rpc
-			// send EngineCommands to the background block authorship task.
-			ManualSealApi::to_delegate(ManualSeal::new(command_sink)),
-		);
+	match command_sink {
+		Some(command_sink) => {
+			io.extend_with(
+				// We provide the rpc handler with the sending end of the channel to allow the rpc
+				// send EngineCommands to the background block authorship task.
+				ManualSealApi::to_delegate(ManualSeal::new(command_sink)),
+			);
+		}
+		_ => {}
 	}
 
 	io