Upgradeable validation functions (#918)

* upgrade primitives to allow changing validation function

* set up storage schema for old parachains code

* fix compilation errors

* fix test compilation

* add some tests for past code meta

* most of the runtime logic for code upgrades

* implement old-code pruning

* add a couple tests

* clean up remaining TODOs

* add a whole bunch of tests for runtime functionality

* remove unused function

* fix runtime compilation

* extract some primitives to parachain crate

* add validation-code upgrades to validation params and result

* extend validation params with code upgrade fields

* provide maximums to validation params

* port test-parachains

* add a code-upgrader test-parachain and tests

* fix collator tests

* move test-parachains to own folder to work around compilation errors

* fix test compilation

* update the Cargo.lock

* fix parachains tests

* remove dbg! invocation

* use new pool in code-upgrader

* bump lockfile

* link TODO to issue

Cargo.toml

@@ -41,13 +41,15 @@ members = [
 	"service",
 	"validation",
 
-	"test-parachains/adder",
-	"test-parachains/adder/collator",
+	"parachain/test-parachains",
+	"parachain/test-parachains/adder",
+	"parachain/test-parachains/adder/collator",
+	"parachain/test-parachains/code-upgrader",
 ]
 exclude = [
 	"runtime/polkadot/wasm",
 	"runtime/kusama/wasm",
-	"test-parachains/adder/wasm",
+	"parachain/test-parachains/adder/wasm",
 ]
 
 [badges]

collator/src/lib.rs

@@ -59,7 +59,7 @@ use polkadot_primitives::{
 	BlockId, Hash, Block,
 	parachain::{
 		self, BlockData, DutyRoster, HeadData, Id as ParaId,
-		PoVBlock, ValidatorId, CollatorPair, LocalValidationData
+		PoVBlock, ValidatorId, CollatorPair, LocalValidationData, GlobalValidationSchedule,
 	}
 };
 use polkadot_cli::{
@@ -154,14 +154,16 @@ pub trait ParachainContext: Clone {
 	fn produce_candidate(
 		&mut self,
 		relay_parent: Hash,
-		status: LocalValidationData,
+		global_validation: GlobalValidationSchedule,
+		local_validation: LocalValidationData,
 	) -> Self::ProduceCandidate;
 }
 
 /// Produce a candidate for the parachain, with given contexts, parent head, and signing key.
 pub async fn collate<P>(
 	relay_parent: Hash,
 	local_id: ParaId,
+	global_validation: GlobalValidationSchedule,
 	local_validation_data: LocalValidationData,
 	mut para_context: P,
 	key: Arc<CollatorPair>,
@@ -173,6 +175,7 @@ pub async fn collate<P>(
 {
 	let (block_data, head_data) = para_context.produce_candidate(
 		relay_parent,
+		global_validation,
 		local_validation_data,
 	).map_err(Error::Collator).await?;
 
@@ -281,6 +284,7 @@ fn build_collator_service<S, P, Extrinsic>(
 
 			let work = future::lazy(move |_| {
 				let api = client.runtime_api();
+				let global_validation = try_fr!(api.global_validation_schedule(&id));
 				let local_validation = match try_fr!(api.local_validation_data(&id, para_id)) {
 					Some(local_validation) => local_validation,
 					None => return future::Either::Left(future::ok(())),
@@ -297,6 +301,7 @@ fn build_collator_service<S, P, Extrinsic>(
 				let collation_work = collate(
 					relay_parent,
 					para_id,
+					global_validation,
 					local_validation,
 					parachain_context,
 					key,
@@ -427,6 +432,7 @@ mod tests {
 		fn produce_candidate(
 			&mut self,
 			_relay_parent: Hash,
+			_global: GlobalValidationSchedule,
 			_local_validation: LocalValidationData,
 		) -> Self::ProduceCandidate {
 			// send messages right back.

parachain/Cargo.toml

@@ -6,13 +6,18 @@ description = "Types and utilities for creating and working with parachains"
 edition = "2018"
 
 [dependencies]
-codec = { package = "parity-scale-codec", version = "1.3.0", default-features = false, features = [ "derive" ] }
-derive_more = { version = "0.99.2", optional = true }
-serde = { version = "1.0.102", default-features = false, features = [ "derive" ], optional = true }
+# note: special care is taken to avoid inclusion of `sp-io` externals when compiling
+# this crate for WASM. This is critical to avoid forcing all parachain WASM into implementing
+# various unnecessary Substrate-specific endpoints.
+codec = { package = "parity-scale-codec", version = "1.1.0", default-features = false, features = [ "derive" ] }
 sp-std = { package = "sp-std", git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
-sp-runtime-interface = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
 sp-wasm-interface = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }
+
+# all optional crates.
+derive_more = { version = "0.99.2", optional = true }
+serde = { version = "1.0.102", default-features = false, features = [ "derive" ], optional = true }
+sp-runtime-interface = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true, default-features = false }
 sp-externalities = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
 sp-io = { git = "https://github.com/paritytech/substrate", branch = "master", optional = true }
@@ -22,14 +27,9 @@ log = { version = "0.4.8", optional = true }
 [target.'cfg(not(target_os = "unknown"))'.dependencies]
 shared_memory = { version = "0.10.0", optional = true }
 
-[dev-dependencies]
-tiny-keccak = "1.5.0"
-adder = { path = "../test-parachains/adder" }
-halt = { path = "../test-parachains/halt" }
-
 [features]
 default = ["std"]
-wasm-api = []
+wasm-api = ["sp-runtime-interface"]
 std = [
 	"codec/std",
 	"derive_more",
@@ -39,6 +39,7 @@ std = [
 	"sp-core/std",
 	"parking_lot",
 	"log",
+	"sp-runtime-interface",
 	"sp-runtime-interface/std",
 	"sp-externalities",
 	"sc-executor",