Shutdown cluster cleanly when SIGTERM or SIGINT is received

Previously, if a developer working with Wallaroo sent a SIGINT via
ctrl-c in the terminal, Wallaroo would exit but wouldn't gracefully
shutdown. This could lead to errors on the next application startup as
recovery files weren't dispose of properly.

With this commit, sending SIGTERM or SIGINT will shut down Wallaroo
gracefully in the same fashion that the external "shutdown cluster"
command work.

Once we have "shrink to fit" in place, the plan would be to have SIGTERM
and SIGINT remove that individual worker from the cluster rathter than
shut the entire cluster down. Once we have shrink to fit in place, we
might change our mind and go in a different direction and leave this
functionality as is.

Closes #1701

lib/wallaroo/startup.pony

@@ -22,11 +22,13 @@ use "files"
 use "itertools"
 use "net"
 use "net/http"
+use "signals"
 use "time"
 use "wallaroo_labs/hub"
 use "wallaroo_labs/mort"
 use "wallaroo_labs/options"
 use "wallaroo/core/boundary"
+use "wallaroo/core/common/"
 use "wallaroo/ent/data_receiver"
 use "wallaroo/ent/cluster_manager"
 use "wallaroo/ent/network"
@@ -319,6 +321,8 @@ actor Startup
         _startup_options.log_rotation where recovery_file_cleaner = this)
       _connections = connections
 
+      _setup_shutdown_handler(connections, this, auth)
+
       let data_receivers = DataReceivers(auth, connections,
         _startup_options.worker_name, is_recovering)
 
@@ -468,6 +472,8 @@ actor Startup
         _startup_options.log_rotation where recovery_file_cleaner = this)
       _connections = connections
 
+      _setup_shutdown_handler(connections, this, auth)
+
       let data_receivers = DataReceivers(auth, connections,
         _startup_options.worker_name)
 
@@ -634,3 +640,30 @@ actor Startup
       "the Wallaroo Community License at https://github.com/WallarooLabs/" +
       "wallaroo/blob/master/LICENSE.md for details, and also please visit " +
       "the page at http://www.wallaroolabs.com/pricing****\n").cstring())
+
+  fun ref _setup_shutdown_handler(c: Connections, r: RecoveryFileCleaner,
+    a: AmbientAuth)
+  =>
+    SignalHandler(WallarooShutdownHandler(c, r, a), Sig.int())
+    SignalHandler(WallarooShutdownHandler(c, r, a), Sig.term())
+
+class WallarooShutdownHandler is SignalNotify
+  """
+  Shutdown gracefully on SIGTERM and SIGINT
+  """
+  let _connections: Connections
+  let _recovery_file_cleaner: RecoveryFileCleaner
+  let _auth: AmbientAuth
+
+  new iso create(c: Connections, r: RecoveryFileCleaner, a: AmbientAuth) =>
+    _connections = c
+    _recovery_file_cleaner = r
+    _auth = a
+
+  fun ref apply(count: U32): Bool =>
+    try
+      let clean_shutdown_msg = ChannelMsgEncoder.clean_shutdown(_auth)?
+      _connections.send_control_to_cluster(clean_shutdown_msg)
+      _recovery_file_cleaner.clean_recovery_files()
+    end
+    false

machida/main.pony

@@ -33,7 +33,7 @@ actor Main
     Machida.start_python()
 
     try
-      SignalHandler(ShutdownHandler, Sig.int())
+      SignalHandler(MachidaShutdownHandler, Sig.int())
 
       var module_name: String = ""
 

machida/signal_handling.pony

@@ -20,7 +20,7 @@ use "signals"
 
 use @Py_Exit[None]()
 
-class iso ShutdownHandler is SignalNotify
+class iso MachidaShutdownHandler is SignalNotify
   fun ref apply(count: U32): Bool =>
     @Py_Exit()
     false