Skip to content

Commit

Permalink
Qemu driver: add graceful shutdown feature
Browse files Browse the repository at this point in the history
  • Loading branch information
Matt Mercer committed Oct 18, 2017
1 parent 0bf7c1f commit 216285c
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 49 deletions.
99 changes: 83 additions & 16 deletions client/driver/qemu.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"log"
"net"
"os"
"os/exec"
"path/filepath"
Expand All @@ -13,7 +14,8 @@ import (
"strings"
"time"

"github.com/hashicorp/go-plugin"
"github.com/coreos/go-semver/semver"
plugin "github.com/hashicorp/go-plugin"
"github.com/hashicorp/nomad/client/config"
"github.com/hashicorp/nomad/client/driver/executor"
dstructs "github.com/hashicorp/nomad/client/driver/structs"
Expand All @@ -29,9 +31,14 @@ var (
)

const (
// The key populated in Node Attributes to indicate presence of the Qemu
// driver
qemuDriverAttr = "driver.qemu"
// The key populated in Node Attributes to indicate presence of the Qemu driver
qemuDriverAttr = "driver.qemu"
qemuDriverVersionAttr = "driver.qemu.version"
qemuDriverLongMonitorPathAttr = "driver.qemu.longsocketpaths"
// Represents an ACPI shutdown request to the VM (emulates pressing a physical power button)
// Reference: https://en.wikibooks.org/wiki/QEMU/Monitor
qemuGracefulShutdownMsg = "system_powerdown\n"
legacyMaxMonitorPathLen = 108
)

// QemuDriver is a driver for running images via Qemu
Expand All @@ -45,17 +52,19 @@ type QemuDriver struct {
}

type QemuDriverConfig struct {
ImagePath string `mapstructure:"image_path"`
Accelerator string `mapstructure:"accelerator"`
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
Args []string `mapstructure:"args"` // extra arguments to qemu executable
ImagePath string `mapstructure:"image_path"`
Accelerator string `mapstructure:"accelerator"`
GracefulShutdown bool `mapstructure:"graceful_shutdown"`
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports.
Args []string `mapstructure:"args"` // extra arguments to qemu executable
}

// qemuHandle is returned from Start/Open as a handle to the PID
type qemuHandle struct {
pluginClient *plugin.Client
userPid int
executor executor.Executor
monitorPath string
killTimeout time.Duration
maxKillTimeout time.Duration
logger *log.Logger
Expand All @@ -64,6 +73,13 @@ type qemuHandle struct {
doneCh chan struct{}
}

func getMonitorPath(dir string, longPathSupport string) (string, error) {
if len(dir) > legacyMaxMonitorPathLen && longPathSupport != "1" {
return "", fmt.Errorf("monitor path is too long")
}
return fmt.Sprintf("%s/qemu-monitor.sock", dir), nil
}

// NewQemuDriver is used to create a new exec driver
func NewQemuDriver(ctx *DriverContext) Driver {
return &QemuDriver{DriverContext: *ctx}
Expand All @@ -81,6 +97,10 @@ func (d *QemuDriver) Validate(config map[string]interface{}) error {
"accelerator": {
Type: fields.TypeString,
},
"graceful_shutdown": {
Type: fields.TypeBool,
Required: false,
},
"port_map": {
Type: fields.TypeArray,
},
Expand Down Expand Up @@ -129,7 +149,23 @@ func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool,
}

node.Attributes[qemuDriverAttr] = "1"
node.Attributes["driver.qemu.version"] = matches[1]
node.Attributes[qemuDriverVersionAttr] = matches[1]

// Prior to qemu 2.10.1, monitor socket paths are truncated to 108 bytes.
// We should consider this if driver.qemu.version is < 2.10.1 and the
// generated monitor path is too long.
//
// Relevant fix is here:
// https://github.com/qemu/qemu/commit/ad9579aaa16d5b385922d49edac2c96c79bcfb6
currentVer := semver.New(matches[1])
fixedSocketPathLenVer := semver.New("2.10.1")
if currentVer.LessThan(*fixedSocketPathLenVer) {
node.Attributes[qemuDriverLongMonitorPathAttr] = "0"
d.logger.Printf("[DEBUG] driver.qemu - long socket paths are not available in this version of QEMU")
} else {
d.logger.Printf("[DEBUG] driver.qemu - long socket paths available in this version of QEMU")
node.Attributes[qemuDriverLongMonitorPathAttr] = "1"
}
return true, nil
}

Expand Down Expand Up @@ -190,6 +226,19 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
"-nographic",
}

var monitorPath string
if d.driverConfig.GracefulShutdown {
// This socket will be used to manage the virtual machine (for example,
// to perform graceful shutdowns)
monitorPath, err = getMonitorPath(ctx.TaskDir.Dir, ctx.TaskEnv.NodeAttrs[qemuDriverLongMonitorPathAttr])
if err == nil {
d.logger.Printf("[DEBUG] driver.qemu - got monitor path OK: %s", monitorPath)
args = append(args, "-monitor", fmt.Sprintf("unix:%s,server,nowait", monitorPath))
} else {
d.logger.Printf("[WARN] driver.qemu - %s", err)
}
}

// Add pass through arguments to qemu executable. A user can specify
// these arguments in driver task configuration. These arguments are
// passed directly to the qemu driver as command line options.
Expand Down Expand Up @@ -239,7 +288,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
)
}

d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " "))
d.logger.Printf("[DEBUG] driver.qemu - starting QemuVM command: %q", strings.Join(args, " "))
pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
executorConfig := &dstructs.ExecutorConfig{
LogFile: pluginLogFile,
Expand Down Expand Up @@ -272,7 +321,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
pluginClient.Kill()
return nil, err
}
d.logger.Printf("[INFO] Started new QemuVM: %s", vmID)
d.logger.Printf("[INFO] driver.qemu - started new QemuVM: %s", vmID)

// Create and Return Handle
maxKill := d.DriverContext.config.MaxKillTimeout
Expand All @@ -282,6 +331,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse
userPid: ps.Pid,
killTimeout: GetKillTimeout(task.KillTimeout, maxKill),
maxKillTimeout: maxKill,
monitorPath: monitorPath,
version: d.config.Version.VersionNumber(),
logger: d.logger,
doneCh: make(chan struct{}),
Expand Down Expand Up @@ -317,7 +367,7 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro

exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
if err != nil {
d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid")
d.logger.Println("[ERR] driver.qemu - error connecting to plugin so destroying plugin pid and user pid")
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil {
d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e)
}
Expand Down Expand Up @@ -381,9 +431,26 @@ func (h *qemuHandle) Signal(s os.Signal) error {
return fmt.Errorf("Qemu driver can't send signals")
}

// TODO: allow a 'shutdown_command' that can be executed over a ssh connection
// to the VM
func (h *qemuHandle) Kill() error {
// If a graceful shutdown was requested at task start time,
// monitorPath will not be empty
if h.monitorPath != "" {
monitorSocket, err := net.Dial("unix", h.monitorPath)
if err == nil {
defer monitorSocket.Close()
h.logger.Printf("[DEBUG] driver.qemu - sending graceful shutdown command to qemu monitor socket at %s", h.monitorPath)
_, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg))
if err == nil {
return nil
}
h.logger.Printf("[WARN] driver.qemu - failed to send '%s' to monitor socket '%s': %s", qemuGracefulShutdownMsg, h.monitorPath, err)
} else {
// OK, that didn't work - we'll continue on and
// attempt to kill the process as a last resort
h.logger.Printf("[WARN] driver.qemu - could not connect to qemu monitor at %s: %s", h.monitorPath, err)
}
}

if err := h.executor.ShutDown(); err != nil {
if h.pluginClient.Exited() {
return nil
Expand All @@ -395,13 +462,13 @@ func (h *qemuHandle) Kill() error {
case <-h.doneCh:
return nil
case <-time.After(h.killTimeout):
h.logger.Printf("[DEBUG] driver.qemu - kill timeout exceeded")
if h.pluginClient.Exited() {
return nil
}
if err := h.executor.Exit(); err != nil {
return fmt.Errorf("executor Exit failed: %v", err)
}

return nil
}
}
Expand All @@ -414,7 +481,7 @@ func (h *qemuHandle) run() {
ps, werr := h.executor.Wait()
if ps.ExitCode == 0 && werr != nil {
if e := killProcess(h.userPid); e != nil {
h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e)
h.logger.Printf("[ERR] driver.qemu - error killing user process: %v", e)
}
}
close(h.doneCh)
Expand Down
107 changes: 77 additions & 30 deletions client/driver/qemu_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ import (
ctestutils "github.com/hashicorp/nomad/client/testutil"
)

func generateString(length int) string {
var newString string
for i := 0; i < length; i++ {
newString = newString + "x"
}
return string(newString)
}

// The fingerprinter test should always pass, even if QEMU is not installed.
func TestQemuDriver_Fingerprint(t *testing.T) {
if !testutil.IsTravis() {
Expand All @@ -39,12 +47,15 @@ func TestQemuDriver_Fingerprint(t *testing.T) {
if !apply {
t.Fatalf("should apply")
}
if node.Attributes["driver.qemu"] == "" {
if node.Attributes[qemuDriverAttr] == "" {
t.Fatalf("Missing Qemu driver")
}
if node.Attributes["driver.qemu.version"] == "" {
if node.Attributes[qemuDriverVersionAttr] == "" {
t.Fatalf("Missing Qemu driver version")
}
if node.Attributes[qemuDriverLongMonitorPathAttr] == "" {
t.Fatalf("Missing Qemu long monitor socket path support flag")
}
}

func TestQemuDriver_StartOpen_Wait(t *testing.T) {
Expand All @@ -56,8 +67,9 @@ func TestQemuDriver_StartOpen_Wait(t *testing.T) {
Name: "linux",
Driver: "qemu",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"graceful_shutdown": true,
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
Expand All @@ -80,15 +92,19 @@ func TestQemuDriver_StartOpen_Wait(t *testing.T) {
}

ctx := testDriverContexts(t, task)
if ctx.DriverCtx.config.Options["qemu.monitorPath"] == "" {
t.Fatalf("Did not expect qemu monitor path to be empty")
}
defer ctx.AllocDir.Destroy()
d := NewQemuDriver(ctx.DriverCtx)

// Copy the test image into the task's directory
dst := ctx.ExecCtx.TaskDir.Dir

copyFile("./test-resources/qemu/linux-0.2.img", filepath.Join(dst, "linux-0.2.img"), t)

if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
t.Fatalf("Prestart faild: %v", err)
t.Fatalf("Prestart failed: %v", err)
}

resp, err := d.Start(ctx.ExecCtx, task)
Expand Down Expand Up @@ -121,34 +137,36 @@ func TestQemuDriverUser(t *testing.T) {
t.Parallel()
}
ctestutils.QemuCompatible(t)
tasks := []*structs.Task{{
Name: "linux",
Driver: "qemu",
User: "alice",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
}},
"args": []string{"-nodefconfig", "-nodefaults"},
"msg": "unknown user alice",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 500,
MemoryMB: 512,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
tasks := []*structs.Task{
{
Name: "linux",
Driver: "qemu",
User: "alice",
Config: map[string]interface{}{
"image_path": "linux-0.2.img",
"accelerator": "tcg",
"graceful_shutdown": false,
"port_map": []map[string]int{{
"main": 22,
"web": 8080,
}},
"args": []string{"-nodefconfig", "-nodefaults"},
"msg": "unknown user alice",
},
LogConfig: &structs.LogConfig{
MaxFiles: 10,
MaxFileSizeMB: 10,
},
Resources: &structs.Resources{
CPU: 500,
MemoryMB: 512,
Networks: []*structs.NetworkResource{
{
ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
},
},
},
},
},
{
Name: "linux",
Driver: "qemu",
Expand Down Expand Up @@ -193,9 +211,38 @@ func TestQemuDriverUser(t *testing.T) {
resp.Handle.Kill()
t.Fatalf("Should've failed")
}

if ctx.DriverCtx.config.Options["qemu.monitorPath"] != "" {
t.Fatalf("Expect qemu monitor path to be empty")
}

msg := task.Config["msg"].(string)
if !strings.Contains(err.Error(), msg) {
t.Fatalf("Expecting '%v' in '%v'", msg, err)
}
}
}

func TestQemuDriverGetMonitorPath(t *testing.T) {
shortPath := generateString(10)
_, err := getMonitorPath(shortPath, "0")
if err != nil {
t.Fatal("Should not have returned an error")
}

longPath := generateString(legacyMaxMonitorPathLen + 100)
_, err = getMonitorPath(longPath, "0")
if err == nil {
t.Fatal("Should have returned an error")
}
_, err = getMonitorPath(longPath, "1")
if err != nil {
t.Fatal("Should not have returned an error")
}

maxLengthPath := generateString(legacyMaxMonitorPathLen)
_, err = getMonitorPath(maxLengthPath, "0")
if err != nil {
t.Fatal("Should not have returned an error")
}
}

0 comments on commit 216285c

Please sign in to comment.