Graceful shutdown for the task manager

client/service/Cargo.toml

@@ -90,3 +90,5 @@ substrate-test-runtime-client = { version = "2.0.0-rc4", path = "../../test-util
 sp-consensus-babe = { version = "0.8.0-rc4", path = "../../primitives/consensus/babe" }
 grandpa = { version = "0.8.0-rc4", package = "sc-finality-grandpa", path = "../finality-grandpa" }
 grandpa-primitives = { version = "2.0.0-rc4", package = "sp-finality-grandpa", path = "../../primitives/finality-grandpa" }
+tokio = { version = "0.2", default-features = false }
+async-std = { version = "1.6", default-features = false }

client/service/src/config.rs

@@ -263,7 +263,9 @@ impl std::convert::From<PathBuf> for BasePath {
 	}
 }
 
-type TaskExecutorInner = Arc<dyn Fn(Pin<Box<dyn Future<Output = ()> + Send>>, TaskType) + Send + Sync>;
+// NOTE: here for code readability.
+pub(crate) type SomeFuture = Pin<Box<dyn Future<Output = ()> + Send>>;
+pub(crate) type JoinFuture = Pin<Box<dyn Future<Output = ()> + Send>>;
 
 /// Callable object that execute tasks.
 ///
@@ -275,56 +277,47 @@ type TaskExecutorInner = Arc<dyn Fn(Pin<Box<dyn Future<Output = ()> + Send>>, Ta
 ///
 /// ```
 /// # use sc_service::TaskExecutor;
-/// # mod tokio { pub mod runtime {
-/// # #[derive(Clone)]
-/// # pub struct Runtime;
-/// # impl Runtime {
-/// # pub fn new() -> Result<Self, ()> { Ok(Runtime) }
-/// # pub fn handle(&self) -> &Self { &self }
-/// # pub fn spawn(&self, _: std::pin::Pin<Box<dyn futures::future::Future<Output = ()> + Send>>) {}
-/// # }
-/// # } }
+/// use futures::future::FutureExt;
 /// use tokio::runtime::Runtime;
 ///
 /// let runtime = Runtime::new().unwrap();
 /// let handle = runtime.handle().clone();
 /// let task_executor: TaskExecutor = (move |future, _task_type| {
-///		handle.spawn(future);
+///		handle.spawn(future).map(|_| ())
 ///	}).into();
 /// ```
 ///
 /// ## Using async-std
 ///
 /// ```
 /// # use sc_service::TaskExecutor;
-/// # mod async_std { pub mod task {
-/// # pub fn spawn(_: std::pin::Pin<Box<dyn futures::future::Future<Output = ()> + Send>>) {}
-/// # } }
 /// let task_executor: TaskExecutor = (|future, _task_type| {
-///		async_std::task::spawn(future);
+///		// NOTE: async-std's JoinHandle is not a Result so we don't need to map the result
+///		async_std::task::spawn(future)
 ///	}).into();
 /// ```
 #[derive(Clone)]
-pub struct TaskExecutor(TaskExecutorInner);
+pub struct TaskExecutor(Arc<dyn Fn(SomeFuture, TaskType) -> JoinFuture + Send + Sync>);
 
 impl std::fmt::Debug for TaskExecutor {
 	fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
 		write!(f, "TaskExecutor")
 	}
 }
 
-impl<F> std::convert::From<F> for TaskExecutor
+impl<F, FUT> std::convert::From<F> for TaskExecutor
 where
-	F: Fn(Pin<Box<dyn Future<Output = ()> + Send>>, TaskType) + Send + Sync + 'static,
+	F: Fn(SomeFuture, TaskType) -> FUT + Send + Sync + 'static,
+	FUT: Future<Output = ()> + Send + 'static,
 {
-	fn from(x: F) -> Self {
-		Self(Arc::new(x))
+	fn from(func: F) -> Self {
+		Self(Arc::new(move |fut, tt| Box::pin(func(fut, tt))))
 	}
 }
 
 impl TaskExecutor {
 	/// Spawns a new asynchronous task.
-	pub fn spawn(&self, future: Pin<Box<dyn Future<Output = ()> + Send>>, task_type: TaskType) {
+	pub fn spawn(&self, future: SomeFuture, task_type: TaskType) -> JoinFuture {
 		self.0(future, task_type)
 	}
 }

client/service/src/task_manager.rs → client/service/src/task_manager/mod.rs

@@ -15,7 +15,7 @@
 
 use std::{panic, result::Result, pin::Pin};
 use exit_future::Signal;
-use log::debug;
+use log::{debug, error};
 use futures::{
 	Future, FutureExt, StreamExt,
 	future::{select, Either, BoxFuture},
@@ -30,16 +30,19 @@ use prometheus_endpoint::{
 };
 use sc_client_api::CloneableSpawn;
 use sp_utils::mpsc::{TracingUnboundedSender, TracingUnboundedReceiver, tracing_unbounded};
-use crate::{config::{TaskExecutor, TaskType}, Error};
+use crate::{config::{TaskExecutor, TaskType, JoinFuture}, Error};
 
 mod prometheus_future;
+#[cfg(test)]
+mod tests;
 
 /// An handle for spawning tasks in the service.
 #[derive(Clone)]
 pub struct SpawnTaskHandle {
 	on_exit: exit_future::Exit,
 	executor: TaskExecutor,
 	metrics: Option<Metrics>,
+	task_notifier: TracingUnboundedSender<JoinFuture>,
 }
 
 impl SpawnTaskHandle {
@@ -67,6 +70,11 @@ impl SpawnTaskHandle {
 		task: impl Future<Output = ()> + Send + 'static,
 		task_type: TaskType,
 	) {
+		if self.task_notifier.is_closed() {
+			debug!("Attempt to spawn a new task has been prevented: {}", name);
+			return;
+		}
+
 		let on_exit = self.on_exit.clone();
 		let metrics = self.metrics.clone();
 
@@ -111,7 +119,16 @@ impl SpawnTaskHandle {
 			}
 		};
 
-		self.executor.spawn(Box::pin(future), task_type);
+		let join_handle = self.executor.spawn(Box::pin(future), task_type);
+		let mut task_notifier = self.task_notifier.clone();
+		self.executor.spawn(
+			Box::pin(async move {
+				if let Err(err) = task_notifier.send(join_handle).await {
+					error!("Could not send spawned task handle to queue: {}", err);
+				}
+			}),
+			TaskType::Async,
+		);
 	}
 }
 
@@ -193,12 +210,12 @@ impl SpawnEssentialTaskHandle {
 		task: impl Future<Output = ()> + Send + 'static,
 		task_type: TaskType,
 	) {
-		let mut essential_failed = self.essential_failed_tx.clone();
+		let essential_failed = self.essential_failed_tx.clone();
 		let essential_task = std::panic::AssertUnwindSafe(task)
 			.catch_unwind()
 			.map(move |_| {
 				log::error!("Essential task `{}` failed. Shutting down service.", name);
-				let _ = essential_failed.send(());
+				let _ = essential_failed.close_channel();
 			});
 
 		let _ = self.inner.spawn_inner(name, essential_task, task_type);
@@ -223,6 +240,8 @@ pub struct TaskManager {
 	essential_failed_rx: TracingUnboundedReceiver<()>,
 	/// Things to keep alive until the task manager is dropped.
 	keep_alive: Box<dyn std::any::Any + Send + Sync>,
+	task_notifier: TracingUnboundedSender<JoinFuture>,
+	completion_future: JoinFuture,
 }
 
 impl TaskManager {
@@ -233,11 +252,18 @@ impl TaskManager {
 		prometheus_registry: Option<&Registry>
 	) -> Result<Self, PrometheusError> {
 		let (signal, on_exit) = exit_future::signal();
+
 		// A side-channel for essential tasks to communicate shutdown.
 		let (essential_failed_tx, essential_failed_rx) = tracing_unbounded("mpsc_essential_tasks");
 
 		let metrics = prometheus_registry.map(Metrics::register).transpose()?;
 
+		let (task_notifier, background_tasks) = tracing_unbounded("mpsc_background_tasks");
+		let completion_future = executor.spawn(
+			Box::pin(background_tasks.for_each_concurrent(None, |x| x)),
+			TaskType::Async,
+		);
+
 		Ok(Self {
 			on_exit,
 			signal: Some(signal),
@@ -246,16 +272,18 @@ impl TaskManager {
 			essential_failed_tx,
 			essential_failed_rx,
 			keep_alive: Box::new(()),
+			task_notifier,
+			completion_future,
 		})
 	}
 
-
 	/// Get a handle for spawning tasks.
 	pub fn spawn_handle(&self) -> SpawnTaskHandle {
 		SpawnTaskHandle {
 			on_exit: self.on_exit.clone(),
 			executor: self.executor.clone(),
 			metrics: self.metrics.clone(),
+			task_notifier: self.task_notifier.clone(),
 		}
 	}
 
@@ -264,19 +292,43 @@ impl TaskManager {
 		SpawnEssentialTaskHandle::new(self.essential_failed_tx.clone(), self.spawn_handle())
 	}
 
-	/// Return a future that will end if an essential task fails.
+	/// Send the signal for termination, prevent new tasks to be created, await for all the existing
+	/// tasks to be finished and drop the object. You can consider this as an async drop.
+	pub fn clean_shutdown(mut self) -> Pin<Box<dyn Future<Output = ()> + Send>> {
+		self.terminate();
+		let keep_alive = self.keep_alive;
+		let completion_future = self.completion_future;
+
+		Box::pin(async move {
+			completion_future.await;
+			drop(keep_alive);
+		})
+	}
+
+	/// Return a future that will end with success if the signal to terminate was sent
+	/// (`self.terminate()`) or with an error if an essential task fails.
+	///
+	/// # Warning
+	///
+	/// This function will not end the remaining task. You must call and await `clean_shutdown()`
+	/// after this.
 	pub fn future<'a>(&'a mut self) -> Pin<Box<dyn Future<Output = Result<(), Error>> + Send + 'a>> {
 		Box::pin(async move {
-			self.essential_failed_rx.next().await;
+			let mut t1 = self.essential_failed_rx.next().fuse();
+			let mut t2 = self.on_exit.clone().fuse();
 
-			Err(Error::Other("Essential task failed.".into()))
+			futures::select! {
+				_ = t1 => Err(Error::Other("Essential task failed.".into())),
+				_ = t2 => Ok(()),
+			}
 		})
 	}
 
 	/// Signal to terminate all the running tasks.
 	pub fn terminate(&mut self) {
 		if let Some(signal) = self.signal.take() {
 			let _ = signal.fire();
+			self.task_notifier.close_channel();
 		}
 	}
 
@@ -286,13 +338,6 @@ impl TaskManager {
 	}
 }
 
-impl Drop for TaskManager {
-	fn drop(&mut self) {
-		debug!(target: "service", "Tasks manager shutdown");
-		self.terminate();
-	}
-}
-
 #[derive(Clone)]
 struct Metrics {
 	// This list is ordered alphabetically