-
Notifications
You must be signed in to change notification settings - Fork 1.9k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Qemu driver: graceful shutdown feature #3411
Changes from 11 commits
00e3cc8
3f6fdfc
16a3614
22f390d
200a12c
1ff9703
2924bad
60030d8
15d7565
1856585
66f9840
f734d84
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,8 +3,10 @@ package driver | |
import ( | ||
"context" | ||
"encoding/json" | ||
"errors" | ||
"fmt" | ||
"log" | ||
"net" | ||
"os" | ||
"os/exec" | ||
"path/filepath" | ||
|
@@ -13,7 +15,8 @@ import ( | |
"strings" | ||
"time" | ||
|
||
"github.com/hashicorp/go-plugin" | ||
"github.com/coreos/go-semver/semver" | ||
plugin "github.com/hashicorp/go-plugin" | ||
"github.com/hashicorp/nomad/client/config" | ||
"github.com/hashicorp/nomad/client/driver/executor" | ||
dstructs "github.com/hashicorp/nomad/client/driver/structs" | ||
|
@@ -26,12 +29,26 @@ import ( | |
|
||
var ( | ||
reQemuVersion = regexp.MustCompile(`version (\d[\.\d+]+)`) | ||
|
||
// Prior to qemu 2.10.1, monitor socket paths are truncated to 108 bytes. | ||
// We should consider this if driver.qemu.version is < 2.10.1 and the | ||
// generated monitor path is too long. | ||
// | ||
// Relevant fix is here: | ||
// https://github.com/qemu/qemu/commit/ad9579aaa16d5b385922d49edac2c96c79bcfb6 | ||
qemuVersionLongSocketPathFix = semver.New("2.10.1") | ||
) | ||
|
||
const ( | ||
// The key populated in Node Attributes to indicate presence of the Qemu | ||
// driver | ||
qemuDriverAttr = "driver.qemu" | ||
// The key populated in Node Attributes to indicate presence of the Qemu driver | ||
qemuDriverAttr = "driver.qemu" | ||
qemuDriverVersionAttr = "driver.qemu.version" | ||
// Represents an ACPI shutdown request to the VM (emulates pressing a physical power button) | ||
// Reference: https://en.wikibooks.org/wiki/QEMU/Monitor | ||
qemuGracefulShutdownMsg = "system_powerdown\n" | ||
qemuMonitorSocketName = "qemu-monitor.sock" | ||
// Maximum socket path length prior to qemu 2.10.1 | ||
qemuLegacyMaxMonitorPathLen = 108 | ||
) | ||
|
||
// QemuDriver is a driver for running images via Qemu | ||
|
@@ -45,17 +62,19 @@ type QemuDriver struct { | |
} | ||
|
||
type QemuDriverConfig struct { | ||
ImagePath string `mapstructure:"image_path"` | ||
Accelerator string `mapstructure:"accelerator"` | ||
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. | ||
Args []string `mapstructure:"args"` // extra arguments to qemu executable | ||
ImagePath string `mapstructure:"image_path"` | ||
Accelerator string `mapstructure:"accelerator"` | ||
GracefulShutdown bool `mapstructure:"graceful_shutdown"` | ||
PortMap []map[string]int `mapstructure:"port_map"` // A map of host port labels and to guest ports. | ||
Args []string `mapstructure:"args"` // extra arguments to qemu executable | ||
} | ||
|
||
// qemuHandle is returned from Start/Open as a handle to the PID | ||
type qemuHandle struct { | ||
pluginClient *plugin.Client | ||
userPid int | ||
executor executor.Executor | ||
monitorPath string | ||
killTimeout time.Duration | ||
maxKillTimeout time.Duration | ||
logger *log.Logger | ||
|
@@ -64,6 +83,29 @@ type qemuHandle struct { | |
doneCh chan struct{} | ||
} | ||
|
||
// getMonitorPath is used to determine whether a qemu monitor socket can be | ||
// safely created and accessed in the task directory by the version of qemu | ||
// present on the host. If it is safe to use, the socket's full path is | ||
// returned along with a nil error. Otherwise, an empty string is returned | ||
// along with a descriptive error. | ||
func (d *QemuDriver) getMonitorPath(dir string) (string, error) { | ||
var longPathSupport bool | ||
currentQemuVer := d.DriverContext.node.Attributes[qemuDriverVersionAttr] | ||
currentQemuSemver := semver.New(currentQemuVer) | ||
if currentQemuSemver.LessThan(*qemuVersionLongSocketPathFix) { | ||
longPathSupport = false | ||
d.logger.Printf("[DEBUG] driver.qemu: long socket paths are not available in this version of QEMU (%s)", currentQemuVer) | ||
} else { | ||
longPathSupport = true | ||
d.logger.Printf("[DEBUG] driver.qemu: long socket paths available in this version of QEMU (%s)", currentQemuVer) | ||
} | ||
fullSocketPath := fmt.Sprintf("%s/%s", dir, qemuMonitorSocketName) | ||
if len(fullSocketPath) > qemuLegacyMaxMonitorPathLen && longPathSupport == false { | ||
return "", fmt.Errorf("monitor path is too long for this version of qemu") | ||
} | ||
return fullSocketPath, nil | ||
} | ||
|
||
// NewQemuDriver is used to create a new exec driver | ||
func NewQemuDriver(ctx *DriverContext) Driver { | ||
return &QemuDriver{DriverContext: *ctx} | ||
|
@@ -81,6 +123,10 @@ func (d *QemuDriver) Validate(config map[string]interface{}) error { | |
"accelerator": { | ||
Type: fields.TypeString, | ||
}, | ||
"graceful_shutdown": { | ||
Type: fields.TypeBool, | ||
Required: false, | ||
}, | ||
"port_map": { | ||
Type: fields.TypeArray, | ||
}, | ||
|
@@ -127,9 +173,11 @@ func (d *QemuDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, | |
delete(node.Attributes, qemuDriverAttr) | ||
return false, fmt.Errorf("Unable to parse Qemu version string: %#v", matches) | ||
} | ||
currentQemuVersion := matches[1] | ||
|
||
node.Attributes[qemuDriverAttr] = "1" | ||
node.Attributes["driver.qemu.version"] = matches[1] | ||
node.Attributes[qemuDriverVersionAttr] = currentQemuVersion | ||
|
||
return true, nil | ||
} | ||
|
||
|
@@ -190,6 +238,19 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse | |
"-nographic", | ||
} | ||
|
||
var monitorPath string | ||
if d.driverConfig.GracefulShutdown { | ||
// This socket will be used to manage the virtual machine (for example, | ||
// to perform graceful shutdowns) | ||
monitorPath, err := d.getMonitorPath(ctx.TaskDir.Dir) | ||
if err != nil { | ||
d.logger.Printf("[ERR] driver.qemu: could not get qemu monitor path - error: %s", err) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. "[ERR] driver.qemu: could not get qemu monitor path: %s" There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done! |
||
return nil, err | ||
} | ||
d.logger.Printf("[DEBUG] driver.qemu: got monitor path OK: %s", monitorPath) | ||
args = append(args, "-monitor", fmt.Sprintf("unix:%s,server,nowait", monitorPath)) | ||
} | ||
|
||
// Add pass through arguments to qemu executable. A user can specify | ||
// these arguments in driver task configuration. These arguments are | ||
// passed directly to the qemu driver as command line options. | ||
|
@@ -239,7 +300,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse | |
) | ||
} | ||
|
||
d.logger.Printf("[DEBUG] Starting QemuVM command: %q", strings.Join(args, " ")) | ||
d.logger.Printf("[DEBUG] driver.qemu: starting QemuVM command: %q", strings.Join(args, " ")) | ||
pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") | ||
executorConfig := &dstructs.ExecutorConfig{ | ||
LogFile: pluginLogFile, | ||
|
@@ -272,7 +333,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse | |
pluginClient.Kill() | ||
return nil, err | ||
} | ||
d.logger.Printf("[INFO] Started new QemuVM: %s", vmID) | ||
d.logger.Printf("[INFO] driver.qemu: started new QemuVM: %s", vmID) | ||
|
||
// Create and Return Handle | ||
maxKill := d.DriverContext.config.MaxKillTimeout | ||
|
@@ -282,6 +343,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse | |
userPid: ps.Pid, | ||
killTimeout: GetKillTimeout(task.KillTimeout, maxKill), | ||
maxKillTimeout: maxKill, | ||
monitorPath: monitorPath, | ||
version: d.config.Version.VersionNumber(), | ||
logger: d.logger, | ||
doneCh: make(chan struct{}), | ||
|
@@ -308,7 +370,7 @@ type qemuId struct { | |
func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { | ||
id := &qemuId{} | ||
if err := json.Unmarshal([]byte(handleID), id); err != nil { | ||
return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) | ||
return nil, fmt.Errorf("Failed to parse handle %q: %v", handleID, err) | ||
} | ||
|
||
pluginConfig := &plugin.ClientConfig{ | ||
|
@@ -317,9 +379,9 @@ func (d *QemuDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, erro | |
|
||
exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) | ||
if err != nil { | ||
d.logger.Println("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid and user pid") | ||
d.logger.Printf("[ERR] driver.qemu: error connecting to plugin so destroying plugin pid %d and user pid %d", id.PluginConfig.Pid, id.UserPid) | ||
if e := destroyPlugin(id.PluginConfig.Pid, id.UserPid); e != nil { | ||
d.logger.Printf("[ERR] driver.qemu: error destroying plugin and userpid: %v", e) | ||
d.logger.Printf("[ERR] driver.qemu: error destroying plugin pid %d and userpid %d: %v", id.PluginConfig.Pid, id.UserPid, e) | ||
} | ||
return nil, fmt.Errorf("error connecting to plugin: %v", err) | ||
} | ||
|
@@ -381,27 +443,34 @@ func (h *qemuHandle) Signal(s os.Signal) error { | |
return fmt.Errorf("Qemu driver can't send signals") | ||
} | ||
|
||
// TODO: allow a 'shutdown_command' that can be executed over a ssh connection | ||
// to the VM | ||
func (h *qemuHandle) Kill() error { | ||
if err := h.executor.ShutDown(); err != nil { | ||
if h.pluginClient.Exited() { | ||
return nil | ||
// First, try sending a graceful shutdown command via the qemu monitor | ||
if err := sendQemuShutdown(h.logger, h.monitorPath, h.userPid); err != nil { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This will log several debug messages relating to graceful shutdown even if the user didn't set the option. We should only attempt the graceful shutdown if the user asked for it. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done! |
||
h.logger.Printf("[DEBUG] driver.qemu: error sending graceful shutdown for user process pid %d: %s", h.userPid, err) | ||
// Issue an interrupt to the qemu process as a last resort | ||
if err := h.executor.ShutDown(); err != nil { | ||
if h.pluginClient.Exited() { | ||
return nil | ||
} | ||
return fmt.Errorf("executor Shutdown failed: %v", err) | ||
} | ||
return fmt.Errorf("executor Shutdown failed: %v", err) | ||
} | ||
|
||
// If the qemu process exits before the kill timeout is reached, doneChan | ||
// will close and we'll exit without an error. If it takes too long, the | ||
// timer will fire and we'll attempt to kill the process. | ||
select { | ||
case <-h.doneCh: | ||
return nil | ||
case <-time.After(h.killTimeout): | ||
h.logger.Printf("[DEBUG] driver.qemu: kill timeout of %s exceeded for user process pid %d", h.killTimeout.String(), h.userPid) | ||
|
||
if h.pluginClient.Exited() { | ||
return nil | ||
} | ||
if err := h.executor.Exit(); err != nil { | ||
return fmt.Errorf("executor Exit failed: %v", err) | ||
} | ||
|
||
return nil | ||
} | ||
} | ||
|
@@ -414,7 +483,7 @@ func (h *qemuHandle) run() { | |
ps, werr := h.executor.Wait() | ||
if ps.ExitCode == 0 && werr != nil { | ||
if e := killProcess(h.userPid); e != nil { | ||
h.logger.Printf("[ERR] driver.qemu: error killing user process: %v", e) | ||
h.logger.Printf("[ERR] driver.qemu: error killing user process pid %d: %v", h.userPid, e) | ||
} | ||
} | ||
close(h.doneCh) | ||
|
@@ -427,3 +496,24 @@ func (h *qemuHandle) run() { | |
h.waitCh <- &dstructs.WaitResult{ExitCode: ps.ExitCode, Signal: ps.Signal, Err: werr} | ||
close(h.waitCh) | ||
} | ||
|
||
// sendQemuShutdown attempts to issue an ACPI power-off command via the qemu | ||
// monitor | ||
func sendQemuShutdown(logger *log.Logger, monitorPath string, userPid int) error { | ||
if monitorPath == "" { | ||
logger.Printf("[DEBUG] driver.qemu: monitorPath not set; will not attempt graceful shutdown for user process pid %d", userPid) | ||
return errors.New("monitorPath not set") | ||
} | ||
monitorSocket, err := net.Dial("unix", monitorPath) | ||
if err != nil { | ||
logger.Printf("[WARN] driver.qemu: could not connect to qemu monitor %q for user process pid %d: %s", monitorPath, userPid, err) | ||
return err | ||
} | ||
defer monitorSocket.Close() | ||
logger.Printf("[DEBUG] driver.qemu: sending graceful shutdown command to qemu monitor socket %q for user process pid %d", monitorPath, userPid) | ||
_, err = monitorSocket.Write([]byte(qemuGracefulShutdownMsg)) | ||
if err != nil { | ||
logger.Printf("[WARN] driver.qemu: failed to send shutdown message %q to monitor socket %q for user process pid %d: %s", qemuGracefulShutdownMsg, monitorPath, userPid, err) | ||
} | ||
return err | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What about windows?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good question! Until today, I didn't realize QEMU was available on Windows hosts. I would need to spin up an EC2 instance and experiment a bit with QEMU monitor interaction on that platform.
It appears an existing QEMU driver option can cause trouble on Windows hosts (
accelerator = "kvm"
). As a first pass, would it be acceptable to fail a task if graceful shutdown has been enabled for the job andruntime.GOOS == "windows"
(ensuring it's called out in documentation, of course)?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yeah I think that is acceptable. Would you mind adding error cases for both graceful and kvm on windows.