Qemu driver: basic testing of graceful shutdown feature

client/driver/qemu.go

@@ -40,6 +40,7 @@ const (
 	// Reference: https://en.wikibooks.org/wiki/QEMU/Monitor
 	qemuGracefulShutdownMsg = "system_powerdown\n"
 	legacyMaxMonitorPathLen = 108
+	qemuMonitorSocketName   = "qemu-monitor.sock"
 )
 
 // QemuDriver is a driver for running images via Qemu
@@ -78,7 +79,7 @@ func getMonitorPath(dir string, longPathSupport string) (string, error) {
 	if len(dir) > legacyMaxMonitorPathLen && longPathSupport != "1" {
 		return "", fmt.Errorf("monitor path is too long")
 	}
-	return fmt.Sprintf("%s/qemu-monitor.sock", dir), nil
+	return fmt.Sprintf("%s/%s", dir, qemuMonitorSocketName), nil
 }
 
 // NewQemuDriver is used to create a new exec driver
@@ -435,9 +436,9 @@ func (h *qemuHandle) Signal(s os.Signal) error {
 
 func (h *qemuHandle) Kill() error {
 	// First, try sending a graceful shutdown command via the qemu monitor
-	err := h.sendQemuShutdown()
+	err := sendQemuShutdown(h.logger, h.monitorPath, h.userPid)
 
-	// If we couldn't send a graceful shutdown via the monitor socket, we'll
+	// If we did not send a graceful shutdown via the monitor socket, we'll
 	// issue an interrupt to the qemu process as a last resort
 	if err != nil {
 		if err := h.executor.ShutDown(); err != nil {
@@ -448,15 +449,14 @@ func (h *qemuHandle) Kill() error {
 		}
 	}
 
-	// At this point, we're waiting for the qemu process to exit. If we've
-	// attempted a graceful shutdown and the guest shuts down in time, doneChan
+	// If the qemu process exits before the kill timeout is reached, doneChan
 	// will close and we'll exit without an error. If it takes too long, the
 	// timer will fire and we'll attempt to kill the process.
 	select {
 	case <-h.doneCh:
 		return nil
 	case <-time.After(h.killTimeout):
-		h.logger.Printf("[DEBUG] driver.qemu: kill timeout exceeded for user process pid %d", h.userPid)
+		h.logger.Printf("[DEBUG] driver.qemu: kill timeout of %s exceeded for user process pid %d", h.killTimeout.String(), h.userPid)
 		if h.pluginClient.Exited() {
 			return nil
 		}
@@ -489,21 +489,24 @@ func (h *qemuHandle) run() {
 	close(h.waitCh)
 }
 
-func (h *qemuHandle) sendQemuShutdown() error {
+func sendQemuShutdown(logger *log.Logger, monitorPath string, userPid int) error {
 	var err error
-	if h.monitorPath == "" {
+	if monitorPath == "" {
+		logger.Printf("[DEBUG] driver.qemu: monitorPath not set; will not attempt graceful shutdown for user process pid %d", userPid)
 		err = errors.New("monitorPath not set")
 	} else {
-		monitorSocket, err := net.Dial("unix", h.monitorPath)
+		var monitorSocket net.Conn
+		monitorSocket, err = net.Dial("unix", monitorPath)
 		if err == nil {
 			defer monitorSocket.Close()
-			h.logger.Printf("[DEBUG] driver.qemu: sending graceful shutdown command to qemu monitor socket %q for user process pid %d", h.monitorPath, h.userPid)
+			logger.Printf("[DEBUG] driver.qemu: sending graceful shutdown command to qemu monitor socket %q for user process pid %d", monitorPath, userPid)
 			_, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg))
 			if err != nil {
-				h.logger.Printf("[WARN] driver.qemu: failed to send shutdown message %q to monitor socket %q for user process pid %d: %s", qemuGracefulShutdownMsg, h.monitorPath, h.userPid, err)
+				logger.Printf("[WARN] driver.qemu: failed to send shutdown message %q to monitor socket %q for user process pid %d: %s", qemuGracefulShutdownMsg, monitorPath, userPid, err)
 			}
+		} else {
+			logger.Printf("[WARN] driver.qemu: could not connect to qemu monitor %q for user process pid %d: %s", monitorPath, userPid, err)
 		}
-		h.logger.Printf("[WARN] driver.qemu: could not connect to qemu monitor %q for user process pid %d: %s", h.monitorPath, h.userPid, err)
 	}
 	return err
 }

client/driver/qemu_test.go

@@ -2,10 +2,12 @@ package driver
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 	"syscall"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -61,7 +63,7 @@ func TestQemuDriver_StartOpen_Wait(t *testing.T) {
 		Config: map[string]interface{}{
 			"image_path":        "linux-0.2.img",
 			"accelerator":       "tcg",
-			"graceful_shutdown": true,
+			"graceful_shutdown": false,
 			"port_map": []map[string]int{{
 				"main": 22,
 				"web":  8080,
@@ -121,6 +123,96 @@ func TestQemuDriver_StartOpen_Wait(t *testing.T) {
 	}
 }
 
+func TestQemuDriver_GracefulShutdown(t *testing.T) {
+	if !testutil.IsTravis() {
+		t.Parallel()
+	}
+	ctestutils.QemuCompatible(t)
+	task := &structs.Task{
+		Name:   "linux",
+		Driver: "qemu",
+		Config: map[string]interface{}{
+			"image_path":        "linux-0.2.img",
+			"accelerator":       "tcg",
+			"graceful_shutdown": true,
+			"port_map": []map[string]int{{
+				"main": 22,
+				"web":  8080,
+			}},
+			"args": []string{"-nodefconfig", "-nodefaults"},
+		},
+		// With the use of tcg acceleration, it's very unlikely a qemu instance
+		// will boot (and gracefully halt) in a reasonable amount of time, so
+		// this timeout is kept low to reduce test execution time
+		KillTimeout: time.Duration(1 * time.Second),
+		LogConfig: &structs.LogConfig{
+			MaxFiles:      10,
+			MaxFileSizeMB: 10,
+		},
+		Resources: &structs.Resources{
+			CPU:      500,
+			MemoryMB: 512,
+			Networks: []*structs.NetworkResource{
+				{
+					ReservedPorts: []structs.Port{{Label: "main", Value: 22000}, {Label: "web", Value: 80}},
+				},
+			},
+		},
+	}
+
+	ctx := testDriverContexts(t, task)
+	defer ctx.AllocDir.Destroy()
+	d := NewQemuDriver(ctx.DriverCtx)
+
+	dst := ctx.ExecCtx.TaskDir.Dir
+
+	copyFile("./test-resources/qemu/linux-0.2.img", filepath.Join(dst, "linux-0.2.img"), t)
+
+	if _, err := d.Prestart(ctx.ExecCtx, task); err != nil {
+		t.Fatalf("Prestart failed: %v", err)
+	}
+
+	resp, err := d.Start(ctx.ExecCtx, task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	// The monitor socket will not exist immediately, so we'll wait up to
+	// 5 seconds for it to become available.
+	monitorPath := fmt.Sprintf("%s/linux/%s", ctx.AllocDir.AllocDir, qemuMonitorSocketName)
+	monitorPathExists := false
+	for i := 0; i < 5; i++ {
+		if _, err := os.Stat(monitorPath); !os.IsNotExist(err) {
+			fmt.Printf("Monitor socket exists at %q\n", monitorPath)
+			monitorPathExists = true
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	if monitorPathExists == false {
+		t.Fatalf("monitor socket did not exist after waiting 5 seconds")
+	}
+
+	// userPid supplied in sendQemuShutdown calls is bogus (it's used only
+	// for log output)
+	if err := sendQemuShutdown(ctx.DriverCtx.logger, "", 0); err == nil {
+		t.Fatalf("sendQemuShutdown should return an error if monitorPath parameter is empty")
+	}
+
+	if err := sendQemuShutdown(ctx.DriverCtx.logger, "/path/that/does/not/exist", 0); err == nil {
+		t.Fatalf("sendQemuShutdown should return an error if file does not exist at monitorPath")
+	}
+
+	if err := sendQemuShutdown(ctx.DriverCtx.logger, monitorPath, 0); err != nil {
+		t.Fatalf("unexpected error from sendQemuShutdown: %s", err)
+	}
+
+	// Clean up
+	if err := resp.Handle.Kill(); err != nil {
+		fmt.Printf("\nError killing Qemu test: %s", err)
+	}
+}
+
 func TestQemuDriverUser(t *testing.T) {
 	if !testutil.IsTravis() {
 		t.Parallel()