Minimal parachain framework part 1 (#113)

* dynamic inclusion threshold calculator

* collators interface

* collation helpers

* initial proposal-creation future

* create proposer when asked to propose

* remove local_availability duty

* statement table tracks includable parachain count

* beginnings of timing future

* finish proposal logic

* remove stray println

* extract shared table to separate module

* change ordering

* includability tracking

* fix doc

* initial changes to parachains module

* initialise dummy block before API calls

* give polkadot control over round proposer based on random seed

* propose only after enough candidates

* flesh out parachains module a bit more

* set_heads

* actually introduce set_heads to runtime

* update block_builder to accept parachains

* split block validity errors from real errors in evaluation

* update WASM runtimes

* polkadot-api methods for parachains additions

* delay evaluation until candidates are ready

* comments

* fix dynamic inclusion with zero initial

* test for includability tracker

* wasm validation of parachain candidates

* move primitives to primitives crate

* remove runtime-std dependency from codec

* adjust doc

* polkadot-parachain-primitives

* kill legacy polkadot-validator crate

* basic-add test chain

* test for basic_add parachain

* move to test-chains dir

* use wasm-build

* new wasm directory layout

* reorganize a bit more

* Fix for rh-minimal-parachain (#141)

* Remove extern "C"

We already encountered such behavior (bug?) in pwasm-std, I believe.

* Fix `panic_fmt` signature by adding `_col`

Wrong `panic_fmt` signature can inhibit some optimizations in LTO mode.

* Add linker flags and use wasm-gc in build script

Pass --import-memory to LLD to emit wasm binary with imported memory.

Also use wasm-gc instead of wasm-build.

* Fix effective_max.

I'm not sure why it was the way it was actually.

* Recompile wasm.

* Fix indent

* more basic_add tests

* validate parachain WASM

* produce statements on receiving statements

* tests for reactive statement production

* fix build

* add OOM lang item to runtime-io

* use dynamic_inclusion when evaluating as well

* fix update_includable_count

* remove dead code

* grumbles

* actually defer round_proposer logic

* update wasm

* address a few more grumbles

* grumbles

* update WASM checkins

* remove dependency on tokio-timer

Cargo.toml

@@ -19,12 +19,13 @@ members = [
 	"polkadot/consensus",
 	"polkadot/executor",
 	"polkadot/keystore",
+	"polkadot/parachain",
 	"polkadot/primitives",
 	"polkadot/runtime",
 	"polkadot/statement-table",
 	"polkadot/transaction-pool",
-	"polkadot/validator",
 	"polkadot/service",
+
 	"substrate/bft",
 	"substrate/client",
 	"substrate/client/db",
@@ -53,6 +54,7 @@ members = [
 	"substrate/serializer",
 	"substrate/state-machine",
 	"substrate/test-runtime",
+
 	"demo/runtime",
 	"demo/primitives",
 	"demo/executor",

demo/runtime/src/lib.rs

@@ -76,6 +76,8 @@ impl consensus::Trait for Concrete {
 pub type Consensus = consensus::Module<Concrete>;
 
 impl timestamp::Trait for Concrete {
+	const SET_POSITION: u32 = 0;
+
 	type Value = u64;
 }
 

polkadot/api/Cargo.toml

@@ -5,7 +5,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
 
 [dependencies]
 error-chain = "0.11"
-log = "0.3"
 polkadot-executor = { path = "../executor" }
 polkadot-runtime = { path = "../runtime" }
 polkadot-primitives = { path = "../primitives" }

polkadot/api/src/lib.rs

@@ -40,8 +40,8 @@ use polkadot_executor::Executor as LocalDispatch;
 use substrate_executor::{NativeExecutionDispatch, NativeExecutor};
 use state_machine::OverlayedChanges;
 use primitives::{AccountId, BlockId, Hash, Index, SessionKey, Timestamp};
-use primitives::parachain::DutyRoster;
-use runtime::{Block, Header, UncheckedExtrinsic, Extrinsic, Call, TimestampCall};
+use primitives::parachain::{DutyRoster, CandidateReceipt, Id as ParaId};
+use runtime::{Block, Header, UncheckedExtrinsic, Extrinsic, Call, TimestampCall, ParachainsCall};
 
 error_chain! {
 	errors {
@@ -135,12 +135,21 @@ pub trait PolkadotApi {
 	/// Get the index of an account at a block.
 	fn index(&self, at: &Self::CheckedBlockId, account: AccountId) -> Result<Index>;
 
+	/// Get the active parachains at a block.
+	fn active_parachains(&self, at: &Self::CheckedBlockId) -> Result<Vec<ParaId>>;
 
-	/// Evaluate a block and see if it gives an error.
-	fn evaluate_block(&self, at: &Self::CheckedBlockId, block: Block) -> Result<()>;
+	/// Get the validation code of a parachain at a block. If the parachain is active, this will always return `Some`.
+	fn parachain_code(&self, at: &Self::CheckedBlockId, parachain: ParaId) -> Result<Option<Vec<u8>>>;
+
+	/// Get the chain head of a parachain. If the parachain is active, this will always return `Some`.
+	fn parachain_head(&self, at: &Self::CheckedBlockId, parachain: ParaId) -> Result<Option<Vec<u8>>>;
+
+	/// Evaluate a block. Returns true if the block is good, false if it is known to be bad,
+	/// and an error if we can't evaluate for some reason.
+	fn evaluate_block(&self, at: &Self::CheckedBlockId, block: Block) -> Result<bool>;
 
 	/// Create a block builder on top of the parent block.
-	fn build_block(&self, parent: &Self::CheckedBlockId, timestamp: Timestamp) -> Result<Self::BlockBuilder>;
+	fn build_block(&self, parent: &Self::CheckedBlockId, timestamp: Timestamp, parachains: Vec<CandidateReceipt>) -> Result<Self::BlockBuilder>;
 }
 
 /// A checked block ID used for the substrate-client implementation of CheckedBlockId;
@@ -213,15 +222,36 @@ impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
 		with_runtime!(self, at, ::runtime::Timestamp::now)
 	}
 
-	fn evaluate_block(&self, at: &CheckedId, block: Block) -> Result<()> {
-		with_runtime!(self, at, || ::runtime::Executive::execute_block(block))
+	fn evaluate_block(&self, at: &CheckedId, block: Block) -> Result<bool> {
+		use substrate_executor::error::ErrorKind as ExecErrorKind;
+
+		let res = with_runtime!(self, at, || ::runtime::Executive::execute_block(block));
+		match res {
+			Ok(()) => Ok(true),
+			Err(err) => match err.kind() {
+				&ErrorKind::Executor(ExecErrorKind::Runtime) => Ok(false),
+				_ => Err(err)
+			}
+		}
 	}
 
 	fn index(&self, at: &CheckedId, account: AccountId) -> Result<Index> {
 		with_runtime!(self, at, || ::runtime::System::account_index(account))
 	}
 
-	fn build_block(&self, parent: &CheckedId, timestamp: Timestamp) -> Result<Self::BlockBuilder> {
+	fn active_parachains(&self, at: &CheckedId) -> Result<Vec<ParaId>> {
+		with_runtime!(self, at, ::runtime::Parachains::active_parachains)
+	}
+
+	fn parachain_code(&self, at: &CheckedId, parachain: ParaId) -> Result<Option<Vec<u8>>> {
+		with_runtime!(self, at, || ::runtime::Parachains::parachain_code(parachain))
+	}
+
+	fn parachain_head(&self, at: &CheckedId, parachain: ParaId) -> Result<Option<Vec<u8>>> {
+		with_runtime!(self, at, || ::runtime::Parachains::parachain_head(parachain))
+	}
+
+	fn build_block(&self, parent: &CheckedId, timestamp: Timestamp, parachains: Vec<CandidateReceipt>) -> Result<Self::BlockBuilder> {
 		let parent = parent.block_id();
 		let header = Header {
 			parent_hash: self.block_hash_from_id(parent)?.ok_or(ErrorKind::UnknownBlock(*parent))?,
@@ -239,6 +269,14 @@ impl<B: Backend> PolkadotApi for Client<B, NativeExecutor<LocalDispatch>>
 					function: Call::Timestamp(TimestampCall::set(timestamp)),
 				},
 				signature: Default::default(),
+			},
+			UncheckedExtrinsic {
+				extrinsic: Extrinsic {
+					signed: Default::default(),
+					index: Default::default(),
+					function: Call::Parachains(ParachainsCall::set_heads(parachains)),
+				},
+				signature: Default::default(),
 			}
 		];
 
@@ -275,7 +313,7 @@ pub struct ClientBlockBuilder<S> {
 impl<S: state_machine::Backend> ClientBlockBuilder<S>
 	where S::Error: Into<client::error::Error>
 {
-	// initialises a block ready to allow extrinsics to be applied.
+	// initialises a block, ready to allow extrinsics to be applied.
 	fn initialise_block(&mut self) -> Result<()> {
 		let result = {
 			let mut ext = state_machine::Ext::new(&mut self.changes, &self.state);
@@ -406,7 +444,7 @@ mod tests {
 		let client = client();
 
 		let id = client.check_id(BlockId::Number(0)).unwrap();
-		let block_builder = client.build_block(&id, 1_000_000).unwrap();
+		let block_builder = client.build_block(&id, 1_000_000, Vec::new()).unwrap();
 		let block = block_builder.bake();
 
 		assert_eq!(block.header.number, 1);

polkadot/collator/Cargo.toml

@@ -1,10 +1,12 @@
 [package]
 name = "polkadot-collator"
 version = "0.1.0"
-authors = ["Parity Technologies <rphmeier@gmail.com>"]
+authors = ["Parity Technologies <admin@parity.io>"]
 description = "Abstract collation logic"
 
 [dependencies]
 futures = "0.1.17"
+substrate-codec = { path = "../../substrate/codec", version = "0.1" }
 substrate-primitives = { path = "../../substrate/primitives", version = "0.1" }
 polkadot-primitives = { path = "../primitives", version = "0.1" }
+polkadot-parachain = { path = "../parachain", version = "0.1" }

polkadot/collator/src/lib.rs

@@ -45,6 +45,7 @@
 //! to be performed, as the collation logic itself.
 
 extern crate futures;
+extern crate substrate_codec as codec;
 extern crate substrate_primitives as primitives;
 extern crate polkadot_primitives;
 
@@ -82,7 +83,6 @@ pub trait RelayChainContext {
 }
 
 /// Collate the necessary ingress queue using the given context.
-// TODO: impl trait
 pub fn collate_ingress<'a, R>(relay_context: R)
 	-> Box<Future<Item=ConsolidatedIngress, Error=R::Error> + 'a>
 	where
@@ -105,7 +105,7 @@ pub fn collate_ingress<'a, R>(relay_context: R)
 	// and then by the parachain ID.
 	//
 	// then transform that into the consolidated egress queue.
-	let future = stream::futures_unordered(egress_fetch)
+	Box::new(stream::futures_unordered(egress_fetch)
 		.fold(BTreeMap::new(), |mut map, (routing_id, egresses)| {
 			for (depth, egress) in egresses.into_iter().rev().enumerate() {
 				let depth = -(depth as i64);
@@ -116,9 +116,7 @@ pub fn collate_ingress<'a, R>(relay_context: R)
 		})
 		.map(|ordered| ordered.into_iter().map(|((_, id), egress)| (id, egress)))
 		.map(|i| i.collect::<Vec<_>>())
-		.map(ConsolidatedIngress);
-
-	Box::new(future)
+		.map(ConsolidatedIngress))
 }
 
 /// Produce a candidate for the parachain.

polkadot/consensus/Cargo.toml

@@ -13,6 +13,7 @@ log = "0.3"
 exit-future = "0.1"
 polkadot-api = { path = "../api" }
 polkadot-collator = { path = "../collator" }
+polkadot-parachain = { path = "../parachain" }
 polkadot-primitives = { path = "../primitives" }
 polkadot-runtime = { path = "../runtime" }
 polkadot-statement-table = { path = "../statement-table" }