Add async test helper to timeout and provide a task_executor automatically

.gitlab-ci.yml

@@ -230,6 +230,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
+    - WASM_BUILD_NO_COLOR=1 SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
 unleash-check:
@@ -621,7 +622,7 @@ deploy-kubernetes-alerting-rules:
     RULES:                         .maintain/monitoring/alerting-rules/alerting-rules.yaml
   script:
     - echo "deploying prometheus alerting rules"
-    - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE} 
+    - kubectl -n ${NAMESPACE} patch prometheusrule ${PROMETHEUSRULE}
         --type=merge --patch "$(sed 's/^/  /;1s/^/spec:\n/' ${RULES})"
   only:
     refs:

Cargo.toml

@@ -171,6 +171,7 @@ members = [
 	"primitives/utils",
 	"primitives/wasm-interface",
 	"test-utils/client",
+	"test-utils/derive",
 	"test-utils/runtime",
 	"test-utils/runtime/client",
 	"test-utils/runtime/transaction-pool",

test-utils/Cargo.toml

@@ -6,6 +6,31 @@ edition = "2018"
 license = "Apache-2.0"
 homepage = "https://substrate.dev"
 repository = "https://github.com/paritytech/substrate/"
+autotests = false
 
 [package.metadata.docs.rs]
 targets = ["x86_64-unknown-linux-gnu"]
+
+[dependencies]
+futures = { version = "0.3.1", features = ["compat"], optional = true }
+substrate-test-utils-derive = { path = "./derive", optional = true }
+tokio = { version = "0.2.13", features = ["macros"], optional = true }
+
+[dev-dependencies]
+sc-service = { path = "../client/service" }
+trybuild = { version = "1.0", features = ["diff"] }
+
+[features]
+macros = [
+	"futures",
+	"substrate-test-utils-derive",
+	"tokio",
+]
+
+[[test]]
+name = "basic"
+path = "tests/basic.rs"
+
+[[test]]
+name = "errors"
+path = "tests/errors.rs"

test-utils/derive/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "substrate-test-utils-derive"
+version = "0.8.0-rc4"
+authors = ["Parity Technologies <admin@parity.io>"]
+edition = "2018"
+license = "Apache-2.0"
+homepage = "https://substrate.dev"
+repository = "https://github.com/paritytech/substrate/"
+
+[dependencies]
+quote = "1"
+syn = { version = "1", default-features = false }
+
+[lib]
+proc-macro = true

test-utils/derive/src/lib.rs

@@ -0,0 +1,96 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use proc_macro::TokenStream;
+use quote::quote;
+
+#[proc_macro_attribute]
+pub fn test(args: TokenStream, item: TokenStream) -> TokenStream {
+	impl_test(args, item)
+}
+
+fn impl_test(args: TokenStream, item: TokenStream) -> TokenStream {
+	let input = syn::parse_macro_input!(item as syn::ItemFn);
+	let args = syn::parse_macro_input!(args as syn::AttributeArgs);
+
+	parse_knobs(input, args).unwrap_or_else(|e| e.to_compile_error().into())
+}
+
+fn parse_knobs(
+	mut input: syn::ItemFn,
+	args: syn::AttributeArgs,
+) -> Result<TokenStream, syn::Error> {
+	let sig = &mut input.sig;
+	let body = &input.block;
+	let attrs = &input.attrs;
+	let vis = input.vis;
+
+	if sig.inputs.len() != 1 {
+		let msg = "the test function accepts only one argument of type sc_service::TaskExecutor";
+		return Err(syn::Error::new_spanned(&sig, msg));
+	}
+	let (task_executor_name, task_executor_type) = match sig.inputs.pop().map(|x| x.into_value()) {
+		Some(syn::FnArg::Typed(x)) => (x.pat, x.ty),
+		_ => {
+			let msg =
+				"the test function accepts only one argument of type sc_service::TaskExecutor";
+			return Err(syn::Error::new_spanned(&sig, msg));
+		}
+	};
+
+	let header = {
+		quote! {
+			#[tokio::test(#(#args)*)]
+		}
+	};
+
+	let result = quote! {
+		#header
+		#(#attrs)*
+		#vis #sig {
+			use substrate_test_utils::futures::future::FutureExt;
+
+			let #task_executor_name: #task_executor_type = (|fut, _| {
+				substrate_test_utils::tokio::spawn(fut).map(drop)
+			})
+			.into();
+			let timeout_task = substrate_test_utils::tokio::time::delay_for(
+				std::time::Duration::from_secs(
+					std::env::var("SUBSTRATE_TEST_TIMEOUT")
+						.ok()
+						.and_then(|x| x.parse().ok())
+						.unwrap_or(600))
+			).fuse();
+			let actual_test_task = async move {
+				#body
+			}
+			.fuse();
+
+			substrate_test_utils::futures::pin_mut!(timeout_task, actual_test_task);
+
+			substrate_test_utils::futures::select! {
+				_ = timeout_task => {
+					panic!("the test took too long");
+				},
+				_ = actual_test_task => {},
+			}
+		}
+	};
+
+	Ok(result.into())
+}

test-utils/src/lib.rs

@@ -17,6 +17,15 @@
 
 //! Test utils
 
+#[doc(hidden)]
+#[cfg(feature = "macros")]
+pub use futures;
+#[cfg(feature = "macros")]
+pub use substrate_test_utils_derive::test;
+#[doc(hidden)]
+#[cfg(feature = "macros")]
+pub use tokio;
+
 /// Panic when the vectors are different, without taking the order into account.
 ///
 /// # Examples

test-utils/tests/basic.rs

@@ -0,0 +1,58 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use sc_service::{TaskExecutor, TaskType};
+
+#[substrate_test_utils::test]
+async fn basic_test(_: TaskExecutor) {
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+#[should_panic(expected = "boo!")]
+async fn panicking_test(_: TaskExecutor) {
+	panic!("boo!");
+}
+
+#[substrate_test_utils::test(max_threads = 2)]
+async fn basic_test_with_args(_: TaskExecutor) {
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+async fn rename_argument(ex: TaskExecutor) {
+	let ex2 = ex.clone();
+	ex2.spawn(Box::pin(async { () }), TaskType::Blocking);
+	assert!(true);
+}
+
+#[substrate_test_utils::test]
+#[should_panic(expected = "test took too long")]
+// NOTE: enable this test only after setting SUBSTRATE_TEST_TIMEOUT to a smaller value
+//
+// SUBSTRATE_TEST_TIMEOUT=1 cargo test -- --ignored timeout
+#[ignore]
+async fn timeout(_: TaskExecutor) {
+	tokio::time::delay_for(std::time::Duration::from_secs(
+		std::env::var("SUBSTRATE_TEST_TIMEOUT")
+			.expect("env var SUBSTRATE_TEST_TIMEOUT has been provided by the user")
+			.parse::<u64>()
+			.unwrap() + 1,
+	))
+	.await;
+}

test-utils/tests/errors.rs

@@ -0,0 +1,24 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[test]
+fn substrate_test_utils_derive_trybuild() {
+	let t = trybuild::TestCases::new();
+	t.compile_fail("tests/missing-func-parameter.rs");
+	t.compile_fail("tests/too-many-func-parameters.rs");
+}

test-utils/tests/missing-func-parameter.rs

@@ -0,0 +1,24 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[substrate_test_utils::test]
+async fn missing_func_parameter() {
+	assert!(true);
+}
+
+fn main() {}

test-utils/tests/missing-func-parameter.stderr

@@ -0,0 +1,5 @@
+error: the test function accepts only one argument of type sc_service::TaskExecutor
+  --> $DIR/missing-func-parameter.rs:20:1
+   |
+20 | async fn missing_func_parameter() {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

test-utils/tests/too-many-func-parameters.rs

@@ -0,0 +1,27 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2020 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[allow(unused_imports)]
+use sc_service::TaskExecutor;
+
+#[substrate_test_utils::test]
+async fn too_many_func_parameters(task_executor_1: TaskExecutor, task_executor_2: TaskExecutor) {
+	assert!(true);
+}
+
+fn main() {}

test-utils/tests/too-many-func-parameters.stderr

@@ -0,0 +1,5 @@
+error: the test function accepts only one argument of type sc_service::TaskExecutor
+  --> $DIR/too-many-func-parameters.rs:23:1
+   |
+23 | async fn too_many_func_parameters(task_executor_1: TaskExecutor, task_executor_2: TaskExecutor) {
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^