From 2129de2906537c749c8a1fecfb5d968a012c32b9 Mon Sep 17 00:00:00 2001 From: Matt Mercer Date: Tue, 17 Oct 2017 17:54:22 -0700 Subject: [PATCH] Qemu driver: add graceful shutdown feature --- client/driver/qemu.go | 99 ++++++++++++++++++---- client/driver/qemu_test.go | 100 ++++++++++++++++------- website/source/docs/drivers/qemu.html.md | 9 +- 3 files changed, 159 insertions(+), 49 deletions(-) diff --git a/client/driver/qemu.go b/client/driver/qemu.go index 1d1f11695f2c..17b137257f41 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "log" + "net" "os" "os/exec" "path/filepath" @@ -13,7 +14,8 @@ import ( "strings" "time" - "github.com/hashicorp/go-plugin" + "github.com/coreos/go-semver/semver" + plugin "github.com/hashicorp/go-plugin" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/executor" dstructs "github.com/hashicorp/nomad/client/driver/structs" @@ -29,9 +31,14 @@ var ( ) const ( - // The key populated in Node Attributes to indicate presence of the Qemu - // driver - qemuDriverAttr = "driver.qemu" + // The key populated in Node Attributes to indicate presence of the Qemu driver + qemuDriverAttr = "driver.qemu" + qemuDriverVersionAttr = "driver.qemu.version" + qemuDriverLongMonitorPathAttr = "driver.qemu.longsocketpaths" + // Represents an ACPI shutdown request to the VM (emulates pressing a physical power button) + // Reference: https://en.wikibooks.org/wiki/QEMU/Monitor + qemuGracefulShutdownMsg = "system_powerdown\n" + legacyMaxMonitorPathLen = 108 ) // QemuDriver is a driver for running images via Qemu @@ -45,10 +52,11 @@ type QemuDriver struct { } type QemuDriverConfig struct { - ImagePath string `mapstructure:"image_path"` - Accelerator string `mapstructure:"accelerator"` - PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. - Args []string `mapstructure:"args"` // extra arguments to qemu executable + ImagePath string `mapstructure:"image_path"` + Accelerator string `mapstructure:"accelerator"` + GracefulShutdown bool `mapstructure:"graceful_shutdown"` + PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. + Args []string `mapstructure:"args"` // extra arguments to qemu executable } // qemuHandle is returned from Start/Open as a handle to the PID @@ -56,6 +64,7 @@ type qemuHandle struct { pluginClient *plugin.Client userPid int executor executor.Executor + monitorPath string killTimeout time.Duration maxKillTimeout time.Duration logger *log.Logger @@ -64,6 +73,13 @@ type qemuHandle struct { doneCh chan struct{} } +func getMonitorPath(dir string, longPathSupport string) (string, error) { + if len(dir) > legacyMaxMonitorPathLen && longPathSupport != "1" { + return "", fmt.Errorf("monitor path is too long") + } + return fmt.Sprintf("%s/qemu-monitor.sock", dir), nil +} + // NewQemuDriver is used to create a new exec driver func NewQemuDriver(ctx *DriverContext) Driver { return &QemuDriver{DriverContext: *ctx} @@ -81,6 +97,10 @@ func (d *QemuDriver) Validate(config map[string]interface{}) error { "accelerator": { Type: fields.TypeString, }, + "graceful_shutdown": { + Type: fields.TypeBool, + Required: false, + }, "port_map": { Type: fields.TypeArray, }, @@ -129,7 +149,23 @@ func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, } node.Attributes[qemuDriverAttr] = "1" - node.Attributes["driver.qemu.version"] = matches[1] + node.Attributes[qemuDriverVersionAttr] = matches[1] + + // Prior to qemu 2.10.1, monitor socket paths are truncated to 108 bytes. + // We should consider this if driver.qemu.version is < 2.10.1 and the + // generated monitor path is too long. + // + // Relevant fix is here: + // https://github.com/qemu/qemu/commit/ad9579aaa16d5b385922d49edac2c96c79bcfb6 + currentVer := semver.New(matches[1]) + fixedSocketPathLenVer := semver.New("2.10.1") + if currentVer.LessThan(*fixedSocketPathLenVer) { + node.Attributes[qemuDriverLongMonitorPathAttr] = "0" + d.logger.Printf("[DEBUG] driver.qemu - long socket paths are not available in this version of QEMU") + } else { + d.logger.Printf("[DEBUG] driver.qemu - long socket paths available in this version of QEMU") + node.Attributes[qemuDriverLongMonitorPathAttr] = "1" + } return true, nil } @@ -190,6 +226,19 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse "-nographic", } + var monitorPath string + if d.driverConfig.GracefulShutdown { + // This socket will be used to manage the virtual machine (for example, + // to perform graceful shutdowns) + monitorPath, err = getMonitorPath(ctx.TaskDir.Dir, ctx.TaskEnv.NodeAttrs[qemuDriverLongMonitorPathAttr]) + if err == nil { + d.logger.Printf("[DEBUG] driver.qemu - got monitor path OK: %s", monitorPath) + args = append(args, "-monitor", fmt.Sprintf("unix:%s,server,nowait", monitorPath)) + } else { + d.logger.Printf("[WARN] driver.qemu - %s", err) + } + } + // Add pass through arguments to qemu executable. A user can specify // these arguments in driver task configuration. These arguments are // passed directly to the qemu driver as command line options. @@ -239,7 +288,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse ) } - d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " ")) + d.logger.Printf("[DEBUG] driver.qemu - starting QemuVM command: %q", strings.Join(args, " ")) pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") executorConfig := &dstructs.ExecutorConfig{ LogFile: pluginLogFile, @@ -272,7 +321,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse pluginClient.Kill() return nil, err } - d.logger.Printf("[INFO] Started new QemuVM: %s", vmID) + d.logger.Printf("[INFO] driver.qemu - started new QemuVM: %s", vmID) // Create and Return Handle maxKill := d.DriverContext.config.MaxKillTimeout @@ -282,6 +331,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse userPid: ps.Pid, killTimeout: GetKillTimeout(task.KillTimeout, maxKill), maxKillTimeout: maxKill, + monitorPath: monitorPath, version: d.config.Version.VersionNumber(), logger: d.logger, doneCh: make(chan struct{}), @@ -317,7 +367,7 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) if err != nil { - d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid") + d.logger.Println("[ERR] driver.qemu - error connecting to plugin so destroying plugin pid and user pid") if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e) } @@ -381,9 +431,26 @@ func (h *qemuHandle) Signal(s os.Signal) error { return fmt.Errorf("Qemu driver can't send signals") } -// TODO: allow a 'shutdown_command' that can be executed over a ssh connection -// to the VM func (h *qemuHandle) Kill() error { + // If a graceful shutdown was requested at task start time, + // monitorPath will not be empty + if h.monitorPath != "" { + monitorSocket, err := net.Dial("unix", h.monitorPath) + if err == nil { + defer monitorSocket.Close() + h.logger.Printf("[DEBUG] driver.qemu - sending graceful shutdown command to qemu monitor socket at %s", h.monitorPath) + _, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg)) + if err == nil { + return nil + } + h.logger.Printf("[WARN] driver.qemu - failed to send '%s' to monitor socket '%s': %s", qemuGracefulShutdownMsg, h.monitorPath, err) + } else { + // OK, that didn't work - we'll continue on and + // attempt to kill the process as a last resort + h.logger.Printf("[WARN] driver.qemu - could not connect to qemu monitor at %s: %s", h.monitorPath, err) + } + } + if err := h.executor.ShutDown(); err != nil { if h.pluginClient.Exited() { return nil @@ -395,13 +462,13 @@ func (h *qemuHandle) Kill() error { case <-h.doneCh: return nil case <-time.After(h.killTimeout): + h.logger.Printf("[DEBUG] driver.qemu - kill timeout exceeded") if h.pluginClient.Exited() { return nil } if err := h.executor.Exit(); err != nil { return fmt.Errorf("executor Exit failed: %v", err) } - return nil } } @@ -414,7 +481,7 @@ func (h *qemuHandle) run() { ps, werr := h.executor.Wait() if ps.ExitCode == 0 && werr != nil { if e := killProcess(h.userPid); e != nil { - h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e) + h.logger.Printf("[ERR] driver.qemu - error killing user process: %v", e) } } close(h.doneCh) diff --git a/client/driver/qemu_test.go b/client/driver/qemu_test.go index 98d491d3de86..646fd3d712f0 100644 --- a/client/driver/qemu_test.go +++ b/client/driver/qemu_test.go @@ -14,6 +14,14 @@ import ( ctestutils "github.com/hashicorp/nomad/client/testutil" ) +func generateString(length int) string { + var newString string + for i := 0; i < length; i++ { + newString = newString + "x" + } + return string(newString) +} + // The fingerprinter test should always pass, even if QEMU is not installed. func TestQemuDriver_Fingerprint(t *testing.T) { if !testutil.IsTravis() { @@ -39,12 +47,15 @@ func TestQemuDriver_Fingerprint(t *testing.T) { if !apply { t.Fatalf("should apply") } - if node.Attributes["driver.qemu"] == "" { + if node.Attributes[qemuDriverAttr] == "" { t.Fatalf("Missing Qemu driver") } - if node.Attributes["driver.qemu.version"] == "" { + if node.Attributes[qemuDriverVersionAttr] == "" { t.Fatalf("Missing Qemu driver version") } + if node.Attributes[qemuDriverLongMonitorPathAttr] == "" { + t.Fatalf("Missing Qemu long monitor socket path support flag") + } } func TestQemuDriver_StartOpen_Wait(t *testing.T) { @@ -56,8 +67,9 @@ func TestQemuDriver_StartOpen_Wait(t *testing.T) { Name: "linux", Driver: "qemu", Config: map[string]interface{}{ - "image_path": "linux-0.2.img", - "accelerator": "tcg", + "image_path": "linux-0.2.img", + "accelerator": "tcg", + "graceful_shutdown": true, "port_map": []map[string]int{{ "main": 22, "web": 8080, @@ -85,10 +97,11 @@ func TestQemuDriver_StartOpen_Wait(t *testing.T) { // Copy the test image into the task's directory dst := ctx.ExecCtx.TaskDir.Dir + copyFile("./test-resources/qemu/linux-0.2.img", filepath.Join(dst, "linux-0.2.img"), t) if _, err := d.Prestart(ctx.ExecCtx, task); err != nil { - t.Fatalf("Prestart faild: %v", err) + t.Fatalf("Prestart failed: %v", err) } resp, err := d.Start(ctx.ExecCtx, task) @@ -121,34 +134,36 @@ func TestQemuDriverUser(t *testing.T) { t.Parallel() } ctestutils.QemuCompatible(t) - tasks := []*structs.Task{{ - Name: "linux", - Driver: "qemu", - User: "alice", - Config: map[string]interface{}{ - "image_path": "linux-0.2.img", - "accelerator": "tcg", - "port_map": []map[string]int{{ - "main": 22, - "web": 8080, - }}, - "args": []string{"-nodefconfig", "-nodefaults"}, - "msg": "unknown user alice", - }, - LogConfig: &structs.LogConfig{ - MaxFiles: 10, - MaxFileSizeMB: 10, - }, - Resources: &structs.Resources{ - CPU: 500, - MemoryMB: 512, - Networks: []*structs.NetworkResource{ - { - ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}}, + tasks := []*structs.Task{ + { + Name: "linux", + Driver: "qemu", + User: "alice", + Config: map[string]interface{}{ + "image_path": "linux-0.2.img", + "accelerator": "tcg", + "graceful_shutdown": false, + "port_map": []map[string]int{{ + "main": 22, + "web": 8080, + }}, + "args": []string{"-nodefconfig", "-nodefaults"}, + "msg": "unknown user alice", + }, + LogConfig: &structs.LogConfig{ + MaxFiles: 10, + MaxFileSizeMB: 10, + }, + Resources: &structs.Resources{ + CPU: 500, + MemoryMB: 512, + Networks: []*structs.NetworkResource{ + { + ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}}, + }, }, }, }, - }, { Name: "linux", Driver: "qemu", @@ -193,9 +208,34 @@ func TestQemuDriverUser(t *testing.T) { resp.Handle.Kill() t.Fatalf("Should've failed") } + msg := task.Config["msg"].(string) if !strings.Contains(err.Error(), msg) { t.Fatalf("Expecting '%v' in '%v'", msg, err) } } } + +func TestQemuDriverGetMonitorPath(t *testing.T) { + shortPath := generateString(10) + _, err := getMonitorPath(shortPath, "0") + if err != nil { + t.Fatal("Should not have returned an error") + } + + longPath := generateString(legacyMaxMonitorPathLen + 100) + _, err = getMonitorPath(longPath, "0") + if err == nil { + t.Fatal("Should have returned an error") + } + _, err = getMonitorPath(longPath, "1") + if err != nil { + t.Fatal("Should not have returned an error") + } + + maxLengthPath := generateString(legacyMaxMonitorPathLen) + _, err = getMonitorPath(maxLengthPath, "0") + if err != nil { + t.Fatal("Should not have returned an error") + } +} diff --git a/website/source/docs/drivers/qemu.html.md b/website/source/docs/drivers/qemu.html.md index 14490de0692e..c26d6e82f7a0 100644 --- a/website/source/docs/drivers/qemu.html.md +++ b/website/source/docs/drivers/qemu.html.md @@ -29,9 +29,10 @@ task "webservice" { driver = "qemu" config { - image_path = "/path/to/my/linux.img" - accelerator = "kvm" - args = ["-nodefaults", "-nodefconfig"] + image_path = "/path/to/my/linux.img" + accelerator = "kvm" + graceful_shutdown = true + args = ["-nodefaults", "-nodefconfig"] } } ``` @@ -47,6 +48,8 @@ The `qemu` driver supports the following configuration in the job spec: If the host machine has `qemu` installed with KVM support, users can specify `kvm` for the `accelerator`. Default is `tcg`. +* `graceful_shutdown` `(bool: false)` - Using the [qemu monitor](https://en.wikibooks.org/wiki/QEMU/Monitor), send an ACPI shutdown signal to virtual machines rather than simply terminating them. This emulates a physical power button press, and gives instances a chance to shut down cleanly. If the VM is still running after ``kill_timeout``, it will be forcefully terminated. (Note that [prior to qemu 2.10.1](https://github.com/qemu/qemu/commit/ad9579aaa16d5b385922d49edac2c96c79bcfb6), the monitor socket path is limited to 108 characters. Graceful shutdown will be disabled if qemu is < 2.10.1 and the generated monitor path exceeds this length. You may encounter this issue if you set long [data_dir](https://www.nomadproject.io/docs/agent/configuration/index.html#data_dir) or [alloc_dir](https://www.nomadproject.io/docs/agent/configuration/client.html#alloc_dir) paths.) + * `port_map` - (Optional) A key-value map of port labels. ```hcl