Merge pull request #247 from hashicorp/f-rkt-enhancements

Rkt Driver Enhancements

client/driver/rkt.go

@@ -7,13 +7,16 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"strings"
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
+	"github.com/hashicorp/nomad/client/driver/args"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -32,7 +35,7 @@ type RktDriver struct {
 // rktHandle is returned from Start/Open as a handle to the PID
 type rktHandle struct {
 	proc   *os.Process
-	name   string
+	image  string
 	logger *log.Logger
 	waitCh chan error
 	doneCh chan struct{}
@@ -41,8 +44,8 @@ type rktHandle struct {
 // rktPID is a struct to map the pid running the process to the vm image on
 // disk
 type rktPID struct {
-	Pid  int
-	Name string
+	Pid   int
+	Image string
 }
 
 // NewRktDriver is used to create a new exec driver
@@ -78,61 +81,99 @@ func (d *RktDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, e
 
 // Run an existing Rkt image.
 func (d *RktDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
-	trust_prefix, ok := task.Config["trust_prefix"]
-	if !ok || trust_prefix == "" {
-		return nil, fmt.Errorf("Missing trust prefix for rkt")
+	// Validate that the config is valid.
+	img, ok := task.Config["image"]
+	if !ok || img == "" {
+		return nil, fmt.Errorf("Missing ACI image for rkt")
 	}
 
+	// Get the tasks local directory.
+	taskName := d.DriverContext.taskName
+	taskDir, ok := ctx.AllocDir.TaskDirs[taskName]
+	if !ok {
+		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
+	}
+	taskLocal := filepath.Join(taskDir, allocdir.TaskLocal)
+
 	// Add the given trust prefix
-	var outBuf, errBuf bytes.Buffer
-	cmd := exec.Command("rkt", "trust", fmt.Sprintf("--prefix=%s", trust_prefix))
-	cmd.Stdout = &outBuf
-	cmd.Stderr = &errBuf
-	d.logger.Printf("[DEBUG] driver.rkt: starting rkt command: %q", cmd.Args)
-	if err := cmd.Run(); err != nil {
-		return nil, fmt.Errorf(
-			"Error running rkt: %s\n\nOutput: %s\n\nError: %s",
-			err, outBuf.String(), errBuf.String())
-	}
-	d.logger.Printf("[DEBUG] driver.rkt: added trust prefix: %q", trust_prefix)
-
-	name, ok := task.Config["name"]
-	if !ok || name == "" {
-		return nil, fmt.Errorf("Missing ACI name for rkt")
-	}
-
-	exec_cmd, ok := task.Config["exec"]
-	if !ok || exec_cmd == "" {
-		d.logger.Printf("[WARN] driver.rkt: could not find a command to execute in the ACI, the default command will be executed")
-	}
-
-	// Run the ACI
-	var aoutBuf, aerrBuf bytes.Buffer
-	run_cmd := []string{
-		"rkt",
-		"run",
-		"--mds-register=false",
-		name,
-	}
-	if exec_cmd != "" {
-		splitted := strings.Fields(exec_cmd)
-		run_cmd = append(run_cmd, "--exec=", splitted[0], "--")
-		run_cmd = append(run_cmd, splitted[1:]...)
-		run_cmd = append(run_cmd, "---")
-	}
-	acmd := exec.Command(run_cmd[0], run_cmd[1:]...)
-	acmd.Stdout = &aoutBuf
-	acmd.Stderr = &aerrBuf
-	d.logger.Printf("[DEBUG] driver:rkt: starting rkt command: %q", acmd.Args)
-	if err := acmd.Start(); err != nil {
-		return nil, fmt.Errorf(
-			"Error running rkt: %s\n\nOutput: %s\n\nError: %s",
-			err, aoutBuf.String(), aerrBuf.String())
-	}
-	d.logger.Printf("[DEBUG] driver.rkt: started ACI: %q", name)
+	trust_prefix, trust_cmd := task.Config["trust_prefix"]
+	if trust_cmd {
+		var outBuf, errBuf bytes.Buffer
+		cmd := exec.Command("rkt", "trust", fmt.Sprintf("--prefix=%s", trust_prefix))
+		cmd.Stdout = &outBuf
+		cmd.Stderr = &errBuf
+		if err := cmd.Run(); err != nil {
+			return nil, fmt.Errorf("Error running rkt trust: %s\n\nOutput: %s\n\nError: %s",
+				err, outBuf.String(), errBuf.String())
+		}
+		d.logger.Printf("[DEBUG] driver.rkt: added trust prefix: %q", trust_prefix)
+	}
+
+	// Build the command.
+	var cmd_args []string
+
+	// Inject the environment variables.
+	envVars := TaskEnvironmentVariables(ctx, task)
+	for k, v := range envVars.Map() {
+		cmd_args = append(cmd_args, fmt.Sprintf("--set-env=%v=%v", k, v))
+	}
+
+	// Disble signature verification if the trust command was not run.
+	if !trust_cmd {
+		cmd_args = append(cmd_args, "--insecure-skip-verify")
+	}
+
+	// Append the run command.
+	cmd_args = append(cmd_args, "run", "--mds-register=false", img)
+
+	// Check if the user has overriden the exec command.
+	if exec_cmd, ok := task.Config["command"]; ok {
+		cmd_args = append(cmd_args, fmt.Sprintf("--exec=%v", exec_cmd))
+	}
+
+	// Add user passed arguments.
+	if userArgs, ok := task.Config["args"]; ok {
+		parsed, err := args.ParseAndReplace(userArgs, envVars.Map())
+		if err != nil {
+			return nil, err
+		}
+
+		// Need to start arguments with "--"
+		if len(parsed) > 0 {
+			cmd_args = append(cmd_args, "--")
+		}
+
+		for _, arg := range parsed {
+			cmd_args = append(cmd_args, fmt.Sprintf("%v", arg))
+		}
+	}
+
+	// Create files to capture stdin and out.
+	stdoutFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stdout", taskName))
+	stderrFilename := filepath.Join(taskLocal, fmt.Sprintf("%s.stderr", taskName))
+
+	stdo, err := os.OpenFile(stdoutFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening file to redirect stdout: %v", err)
+	}
+
+	stde, err := os.OpenFile(stderrFilename, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
+	if err != nil {
+		return nil, fmt.Errorf("Error opening file to redirect stderr: %v", err)
+	}
+
+	cmd := exec.Command("rkt", cmd_args...)
+	cmd.Stdout = stdo
+	cmd.Stderr = stde
+
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("Error running rkt: %v", err)
+	}
+
+	d.logger.Printf("[DEBUG] driver.rkt: started ACI %q with: %v", img, cmd.Args)
 	h := &rktHandle{
-		proc:   acmd.Process,
-		name:   name,
+		proc:   cmd.Process,
+		image:  img,
 		logger: d.logger,
 		doneCh: make(chan struct{}),
 		waitCh: make(chan error, 1),
@@ -158,7 +199,7 @@ func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error
 	// Return a driver handle
 	h := &rktHandle{
 		proc:   proc,
-		name:   qpid.Name,
+		image:  qpid.Image,
 		logger: d.logger,
 		doneCh: make(chan struct{}),
 		waitCh: make(chan error, 1),
@@ -171,8 +212,8 @@ func (d *RktDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error
 func (h *rktHandle) ID() string {
 	// Return a handle to the PID
 	pid := &rktPID{
-		Pid:  h.proc.Pid,
-		Name: h.name,
+		Pid:   h.proc.Pid,
+		Image: h.image,
 	}
 	data, err := json.Marshal(pid)
 	if err != nil {

client/driver/rkt_test.go

@@ -2,10 +2,13 @@ package driver
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
+	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/nomad/structs"
 
@@ -15,13 +18,13 @@ import (
 func TestRktDriver_Handle(t *testing.T) {
 	h := &rktHandle{
 		proc:   &os.Process{Pid: 123},
-		name:   "foo",
+		image:  "foo",
 		doneCh: make(chan struct{}),
 		waitCh: make(chan error, 1),
 	}
 
 	actual := h.ID()
-	expected := `Rkt:{"Pid":123,"Name":"foo"}`
+	expected := `Rkt:{"Pid":123,"Image":"foo"}`
 	if actual != expected {
 		t.Errorf("Expected `%s`, found `%s`", expected, actual)
 	}
@@ -59,8 +62,8 @@ func TestRktDriver_Start(t *testing.T) {
 		Name: "etcd",
 		Config: map[string]string{
 			"trust_prefix": "coreos.com/etcd",
-			"name":         "coreos.com/etcd:v2.0.4",
-			"exec":         "/etcd --version",
+			"image":        "coreos.com/etcd:v2.0.4",
+			"command":      "/etcd",
 		},
 	}
 
@@ -98,8 +101,9 @@ func TestRktDriver_Start_Wait(t *testing.T) {
 		Name: "etcd",
 		Config: map[string]string{
 			"trust_prefix": "coreos.com/etcd",
-			"name":         "coreos.com/etcd:v2.0.4",
-			"exec":         "/etcd --version",
+			"image":        "coreos.com/etcd:v2.0.4",
+			"command":      "/etcd",
+			"args":         "--version",
 		},
 	}
 
@@ -132,3 +136,94 @@ func TestRktDriver_Start_Wait(t *testing.T) {
 		t.Fatalf("timeout")
 	}
 }
+
+func TestRktDriver_Start_Wait_Skip_Trust(t *testing.T) {
+	ctestutils.RktCompatible(t)
+	task := &structs.Task{
+		Name: "etcd",
+		Config: map[string]string{
+			"image":   "coreos.com/etcd:v2.0.4",
+			"command": "/etcd",
+			"args":    "--version",
+		},
+	}
+
+	driverCtx := testDriverContext(task.Name)
+	ctx := testDriverExecContext(task, driverCtx)
+	d := NewRktDriver(driverCtx)
+	defer ctx.AllocDir.Destroy()
+
+	handle, err := d.Start(ctx, task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+	defer handle.Kill()
+
+	// Update should be a no-op
+	err = handle.Update(task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	select {
+	case err := <-handle.WaitCh():
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timeout")
+	}
+}
+
+func TestRktDriver_Start_Wait_Logs(t *testing.T) {
+	ctestutils.RktCompatible(t)
+	task := &structs.Task{
+		Name: "etcd",
+		Config: map[string]string{
+			"trust_prefix": "coreos.com/etcd",
+			"image":        "coreos.com/etcd:v2.0.4",
+			"command":      "/etcd",
+			"args":         "--version",
+		},
+	}
+
+	driverCtx := testDriverContext(task.Name)
+	ctx := testDriverExecContext(task, driverCtx)
+	d := NewRktDriver(driverCtx)
+	defer ctx.AllocDir.Destroy()
+
+	handle, err := d.Start(ctx, task)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if handle == nil {
+		t.Fatalf("missing handle")
+	}
+	defer handle.Kill()
+
+	select {
+	case err := <-handle.WaitCh():
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+	case <-time.After(5 * time.Second):
+		t.Fatalf("timeout")
+	}
+
+	taskDir, ok := ctx.AllocDir.TaskDirs[task.Name]
+	if !ok {
+		t.Fatalf("Could not find task directory for task: %v", task)
+	}
+	stdout := filepath.Join(taskDir, allocdir.TaskLocal, fmt.Sprintf("%v.stdout", task.Name))
+	data, err := ioutil.ReadFile(stdout)
+	if err != nil {
+		t.Fatalf("Failed to read tasks stdout: %v", err)
+	}
+
+	if len(data) == 0 {
+		t.Fatal("Task's stdout is empty")
+	}
+}

website/source/docs/drivers/rkt.html.md

@@ -18,10 +18,13 @@ containers.
 
 The `Rkt` driver supports the following configuration in the job spec:
 
-* `trust_prefix` - **(Required)** The trust prefix to be passed to rkt. Must be reachable from
-the box running the nomad agent.
-* `name` - **(Required)** Fully qualified name of an image to run using rkt
-* `exec` - **(Optional**) A command to execute on the ACI
+* `trust_prefix` - **(Optional)** The trust prefix to be passed to rkt. Must be reachable from
+the box running the nomad agent. If not specified, the image is run without
+verifying the image signature.
+* `image` - **(Required)** The image to run which may be specified by name,
+hash, ACI address or docker registry.
+* `command` - **(Optional**) A command to execute on the ACI.
+* `args` - **(Optional**) A string of args to pass into the image.
 
 ## Client Requirements