Qemu driver: graceful shutdown feature

client/driver/qemu.go

@@ -3,8 +3,10 @@ package driver
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log"
+	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -13,7 +15,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/hashicorp/go-plugin"
+	"github.com/coreos/go-semver/semver"
+	plugin "github.com/hashicorp/go-plugin"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/client/driver/executor"
 	dstructs "github.com/hashicorp/nomad/client/driver/structs"
@@ -29,9 +32,15 @@ var (
 )
 
 const (
-	// The key populated in Node Attributes to indicate presence of the Qemu
-	// driver
-	qemuDriverAttr = "driver.qemu"
+	// The key populated in Node Attributes to indicate presence of the Qemu driver
+	qemuDriverAttr                = "driver.qemu"
+	qemuDriverVersionAttr         = "driver.qemu.version"
+	qemuDriverLongMonitorPathAttr = "driver.qemu.longsocketpaths"
+	// Represents an ACPI shutdown request to the VM (emulates pressing a physical power button)
+	// Reference: https://en.wikibooks.org/wiki/QEMU/Monitor
+	qemuGracefulShutdownMsg = "system_powerdown\n"
+	legacyMaxMonitorPathLen = 108
+	qemuMonitorSocketName   = "qemu-monitor.sock"
 )
 
 // QemuDriver is a driver for running images via Qemu
@@ -45,17 +54,19 @@ type QemuDriver struct {
 }
 
 type QemuDriverConfig struct {
-	ImagePath   string           `mapstructure:"image_path"`
-	Accelerator string           `mapstructure:"accelerator"`
-	PortMap     []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
-	Args        []string         `mapstructure:"args"`     // extra arguments to qemu executable
+	ImagePath        string           `mapstructure:"image_path"`
+	Accelerator      string           `mapstructure:"accelerator"`
+	GracefulShutdown bool             `mapstructure:"graceful_shutdown"`
+	PortMap          []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
+	Args             []string         `mapstructure:"args"`     // extra arguments to qemu executable
 }
 
 // qemuHandle is returned from Start/Open as a handle to the PID
 type qemuHandle struct {
 	pluginClient   *plugin.Client
 	userPid        int
 	executor       executor.Executor
+	monitorPath    string
 	killTimeout    time.Duration
 	maxKillTimeout time.Duration
 	logger         *log.Logger
@@ -64,6 +75,13 @@ type qemuHandle struct {
 	doneCh         chan struct{}
 }
 
+func getMonitorPath(dir string, longPathSupport string) (string, error) {
+	if len(dir) > legacyMaxMonitorPathLen && longPathSupport != "1" {
+		return "", fmt.Errorf("monitor path is too long")
+	}
+	return fmt.Sprintf("%s/%s", dir, qemuMonitorSocketName), nil
+}
+
 // NewQemuDriver is used to create a new exec driver
 func NewQemuDriver(ctx *DriverContext) Driver {
 	return &QemuDriver{DriverContext: *ctx}
@@ -81,6 +99,10 @@ func (d *QemuDriver) Validate(config map[string]interface{}) error {
 			"accelerator": {
 				Type: fields.TypeString,
 			},
+			"graceful_shutdown": {
+				Type:     fields.TypeBool,
+				Required: false,
+			},
 			"port_map": {
 				Type: fields.TypeArray,
 			},
@@ -127,9 +149,26 @@ func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool,
 		delete(node.Attributes, qemuDriverAttr)
 		return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches)
 	}
+	currentQemuVersion := matches[1]
 
 	node.Attributes[qemuDriverAttr] = "1"
-	node.Attributes["driver.qemu.version"] = matches[1]
+	node.Attributes[qemuDriverVersionAttr] = currentQemuVersion
+
+	// Prior to qemu 2.10.1, monitor socket paths are truncated to 108 bytes.
+	// We should consider this if driver.qemu.version is < 2.10.1 and the
+	// generated monitor path is too long.
+	//
+	// Relevant fix is here:
+	// https://github.com/qemu/qemu/commit/ad9579aaa16d5b385922d49edac2c96c79bcfb6
+	currentQemuSemver := semver.New(currentQemuVersion)
+	fixedSocketPathLenVer := semver.New("2.10.1")
+	if currentQemuSemver.LessThan(*fixedSocketPathLenVer) {
+		node.Attributes[qemuDriverLongMonitorPathAttr] = "0"
+		d.logger.Printf("[DEBUG] driver.qemu: long socket paths are not available in this version of QEMU (%s)", currentQemuVersion)
+	} else {
+		d.logger.Printf("[DEBUG] driver.qemu: long socket paths available in this version of QEMU (%s)", currentQemuVersion)
+		node.Attributes[qemuDriverLongMonitorPathAttr] = "1"
+	}
 	return true, nil
 }
 
@@ -190,6 +229,19 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
 		"-nographic",
 	}
 
+	var monitorPath string
+	if d.driverConfig.GracefulShutdown {
+		// This socket will be used to manage the virtual machine (for example,
+		// to perform graceful shutdowns)
+		monitorPath, err = getMonitorPath(ctx.TaskDir.Dir, ctx.TaskEnv.NodeAttrs[qemuDriverLongMonitorPathAttr])
+		if err == nil {
+			d.logger.Printf("[DEBUG] driver.qemu: got monitor path OK: %s", monitorPath)
+			args = append(args, "-monitor", fmt.Sprintf("unix:%s,server,nowait", monitorPath))
+		} else {
+			d.logger.Printf("[WARN] driver.qemu: %s", err)
+		}
+	}
+
 	// Add pass through arguments to qemu executable. A user can specify
 	// these arguments in driver task configuration. These arguments are
 	// passed directly to the qemu driver as command line options.
@@ -239,7 +291,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
 		)
 	}
 
-	d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " "))
+	d.logger.Printf("[DEBUG] driver.qemu: starting QemuVM command: %q", strings.Join(args, " "))
 	pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
 	executorConfig := &dstructs.ExecutorConfig{
 		LogFile:  pluginLogFile,
@@ -272,7 +324,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
 		pluginClient.Kill()
 		return nil, err
 	}
-	d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
+	d.logger.Printf("[INFO] driver.qemu: started new QemuVM: %s", vmID)
 
 	// Create and Return Handle
 	maxKill := d.DriverContext.config.MaxKillTimeout
@@ -282,6 +334,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
 		userPid:        ps.Pid,
 		killTimeout:    GetKillTimeout(task.KillTimeout, maxKill),
 		maxKillTimeout: maxKill,
+		monitorPath:    monitorPath,
 		version:        d.config.Version.VersionNumber(),
 		logger:         d.logger,
 		doneCh:         make(chan struct{}),
@@ -308,7 +361,7 @@ type qemuId struct {
 func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
 	id := &qemuId{}
 	if err := json.Unmarshal([]byte(handleID), id); err != nil {
-		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
+		return nil, fmt.Errorf("Failed to parse handle %q: %v", handleID, err)
 	}
 
 	pluginConfig := &plugin.ClientConfig{
@@ -317,9 +370,9 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro
 
 	exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
 	if err != nil {
-		d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
+		d.logger.Printf("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid %d and user pid %d", id.PluginConfig.Pid, id.UserPid)
 		if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
-			d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e)
+			d.logger.Printf("[ERR] driver.qemu: error destroying plugin pid %d and userpid %d: %v", id.PluginConfig.Pid, id.UserPid, e)
 		}
 		return nil, fmt.Errorf("error connecting to plugin: %v", err)
 	}
@@ -381,27 +434,36 @@ func (h *qemuHandle) Signal(s os.Signal) error {
 	return fmt.Errorf("Qemu driver can't send signals")
 }
 
-// TODO: allow a 'shutdown_command' that can be executed over a ssh connection
-// to the VM
 func (h *qemuHandle) Kill() error {
-	if err := h.executor.ShutDown(); err != nil {
-		if h.pluginClient.Exited() {
-			return nil
+	// First, try sending a graceful shutdown command via the qemu monitor
+	err := sendQemuShutdown(h.logger, h.monitorPath, h.userPid)
+
+	// If we did not send a graceful shutdown via the monitor socket, we'll
+	// issue an interrupt to the qemu process as a last resort
+	if err != nil {
+		if err := h.executor.ShutDown(); err != nil {
+			if h.pluginClient.Exited() {
+				return nil
+			}
+			return fmt.Errorf("executor Shutdown failed: %v", err)
 		}
-		return fmt.Errorf("executor Shutdown failed: %v", err)
 	}
 
+	// If the qemu process exits before the kill timeout is reached, doneChan
+	// will close and we'll exit without an error. If it takes too long, the
+	// timer will fire and we'll attempt to kill the process.
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
+		h.logger.Printf("[DEBUG] driver.qemu: kill timeout of %s exceeded for user process pid %d", h.killTimeout.String(), h.userPid)
+
 		if h.pluginClient.Exited() {
 			return nil
 		}
 		if err := h.executor.Exit(); err != nil {
 			return fmt.Errorf("executor Exit failed: %v", err)
 		}
-
 		return nil
 	}
 }
@@ -414,7 +476,7 @@ func (h *qemuHandle) run() {
 	ps, werr := h.executor.Wait()
 	if ps.ExitCode == 0 && werr != nil {
 		if e := killProcess(h.userPid); e != nil {
-			h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e)
+			h.logger.Printf("[ERR] driver.qemu: error killing user process pid %d: %v", h.userPid, e)
 		}
 	}
 	close(h.doneCh)
@@ -427,3 +489,23 @@ func (h *qemuHandle) run() {
 	h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: werr}
 	close(h.waitCh)
 }
+
+func sendQemuShutdown(logger *log.Logger, monitorPath string, userPid int) error {
+	if monitorPath == "" {
+		logger.Printf("[DEBUG] driver.qemu: monitorPath not set; will not attempt graceful shutdown for user process pid %d", userPid)
+		return errors.New("monitorPath not set")
+	} else {
+		monitorSocket, err := net.Dial("unix", monitorPath)
+		if err == nil {
+			defer monitorSocket.Close()
+			logger.Printf("[DEBUG] driver.qemu: sending graceful shutdown command to qemu monitor socket %q for user process pid %d", monitorPath, userPid)
+			_, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg))
+			if err != nil {
+				logger.Printf("[WARN] driver.qemu: failed to send shutdown message %q to monitor socket %q for user process pid %d: %s", qemuGracefulShutdownMsg, monitorPath, userPid, err)
+			}
+		} else {
+			logger.Printf("[WARN] driver.qemu: could not connect to qemu monitor %q for user process pid %d: %s", monitorPath, userPid, err)
+		}
+		return err
+	}
+}