From f422ef0495030614ec535300a1c52dd30f87808c Mon Sep 17 00:00:00 2001 From: Clint Shryock Date: Thu, 15 Oct 2015 16:40:08 -0500 Subject: [PATCH] drivers: Add/Use go-getter to fetch remote binaries Updates Qemu, Java drivers to use go-getter to fetch binaries Adds remote artifact support for Exec, Raw Exec drivers --- client/driver/exec.go | 39 +++++++++++++- client/driver/exec_test.go | 50 ++++++++++++++++++ client/driver/java.go | 39 +++----------- client/driver/qemu.go | 38 ++++---------- client/driver/raw_exec.go | 44 +++++++++++++--- client/driver/raw_exec_test.go | 54 ++++++++++++++++++++ client/executor/exec_universal.go | 6 ++- website/.ruby-version | 1 + website/source/docs/drivers/exec.html.md | 6 +++ website/source/docs/drivers/java.html.md | 2 +- website/source/docs/drivers/raw_exec.html.md | 7 ++- 11 files changed, 216 insertions(+), 70 deletions(-) create mode 100644 website/.ruby-version diff --git a/client/driver/exec.go b/client/driver/exec.go index f7869e94bc7a..da221f0c9228 100644 --- a/client/driver/exec.go +++ b/client/driver/exec.go @@ -2,10 +2,15 @@ package driver import ( "fmt" + "log" + "path" + "path/filepath" "runtime" "syscall" "time" + "github.com/hashicorp/go-getter" + "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/executor" "github.com/hashicorp/nomad/nomad/structs" @@ -41,12 +46,44 @@ func (d *ExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, } func (d *ExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { - // Get the command + // Get the command to be ran, or if omitted, download an artifact for + // execution. Currently a supplied command takes precedence, and an artifact + // is only downloaded if no command is supplied command, ok := task.Config["command"] if !ok || command == "" { return nil, fmt.Errorf("missing command for exec driver") } + source, ok := task.Config["artifact_source"] + if ok && source != "" { + // Proceed to download an artifact to be executed. + // We use go-getter to support a variety of protocols, but need to change + // file permissions of the resulted download to be executable + + // Create a location to download the artifact. + taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] + if !ok { + return nil, fmt.Errorf("[Err] driver.Exec: Could not find task directory for task: %v", d.DriverContext.taskName) + } + destDir := filepath.Join(taskDir, allocdir.TaskLocal) + + artifactName := path.Base(source) + artifactFile := filepath.Join(destDir, artifactName) + if err := getter.GetFile(artifactFile, source); err != nil { + return nil, fmt.Errorf("[Err] driver.Exec: Error downloading source for Exec driver: %s", err) + } + + // Add execution permissions to the newly downloaded artifact + if runtime.GOOS != "windows" { + if err := syscall.Chmod(artifactFile, 0655); err != nil { + log.Printf("[Err] driver.Exec: Error making artifact executable: %s", err) + } + } + + // re-assign the command to be the local execution path + command = filepath.Join(allocdir.TaskLocal, command) + } + // Get the environment variables. envVars := TaskEnvironmentVariables(ctx, task) diff --git a/client/driver/exec_test.go b/client/driver/exec_test.go index d2c2886524e5..6b943e950ae5 100644 --- a/client/driver/exec_test.go +++ b/client/driver/exec_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "path/filepath" "reflect" + "runtime" "testing" "time" @@ -120,6 +121,55 @@ func TestExecDriver_Start_Wait(t *testing.T) { } } +func TestExecDriver_Start_Artifact_Wait(t *testing.T) { + ctestutils.ExecCompatible(t) + var file string + switch runtime.GOOS { + case "darwin": + file = "hi_darwin_amd64" + default: + file = "hi_linux_amd64" + } + + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file), + "command": file, + }, + Resources: basicResources, + } + + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + d := NewExecDriver(driverCtx) + + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Update should be a no-op + err = handle.Update(task) + if err != nil { + t.Fatalf("err: %v", err) + } + + // Task should terminate quickly + select { + case err := <-handle.WaitCh(): + if err != nil { + t.Fatalf("err: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatalf("timeout") + } +} + func TestExecDriver_Start_Wait_AllocDir(t *testing.T) { ctestutils.ExecCompatible(t) diff --git a/client/driver/java.go b/client/driver/java.go index f5faa06f216f..ac2c3c6f388f 100644 --- a/client/driver/java.go +++ b/client/driver/java.go @@ -3,9 +3,6 @@ package driver import ( "bytes" "fmt" - "io" - "net/http" - "os" "os/exec" "path" "path/filepath" @@ -14,6 +11,7 @@ import ( "syscall" "time" + "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/executor" @@ -97,37 +95,18 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("missing jar source for Java Jar driver") } - // Attempt to download the thing - // Should be extracted to some kind of Http Fetcher - // Right now, assume publicly accessible HTTP url - resp, err := http.Get(source) - if err != nil { - return nil, fmt.Errorf("Error downloading source for Java driver: %s", err) - } - - // Get the tasks local directory. taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] if !ok { return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) - // Create a location to download the binary. - fName := path.Base(source) - fPath := filepath.Join(taskLocal, fName) - f, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666) - if err != nil { - return nil, fmt.Errorf("Error opening file to download to: %s", err) - } - - defer f.Close() - defer resp.Body.Close() + destDir := filepath.Join(taskDir, allocdir.TaskLocal) - // Copy remote file to local directory for execution - // TODO: a retry of sort if io.Copy fails, for large binaries - _, ioErr := io.Copy(f, resp.Body) - if ioErr != nil { - return nil, fmt.Errorf("Error copying jar from source: %s", ioErr) + // Create a location to download the binary. + jarName := path.Base(source) + jarPath := filepath.Join(destDir, jarName) + if err := getter.GetFile(jarPath, source); err != nil { + return nil, fmt.Errorf("Error downloading source for Java driver: %s", err) } // Get the environment variables. @@ -141,10 +120,8 @@ func (d *JavaDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, args = append(args, jvm_options) } - // Build the argument list - args = append(args, "-jar", filepath.Join(allocdir.TaskLocal, fName)) - // Build the argument list. + args = append(args, "-jar", filepath.Join(allocdir.TaskLocal, jarName)) if argRaw, ok := task.Config["args"]; ok { args = append(args, argRaw) } diff --git a/client/driver/qemu.go b/client/driver/qemu.go index 540dc9068a41..739017a6d253 100644 --- a/client/driver/qemu.go +++ b/client/driver/qemu.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "log" - "net/http" "os" "os/exec" "path/filepath" @@ -19,6 +18,7 @@ import ( "syscall" "time" + "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/nomad/structs" @@ -94,45 +94,25 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, return nil, fmt.Errorf("Missing required Task Resource: Memory") } - // Attempt to download the thing - // Should be extracted to some kind of Http Fetcher - // Right now, assume publicly accessible HTTP url - resp, err := http.Get(source) - if err != nil { - return nil, fmt.Errorf("Error downloading source for Qemu driver: %s", err) - } - // Get the tasks local directory. taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] if !ok { return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) } - taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) - // Create a location in the local directory to download and store the image. - // TODO: Caching + // Create a location to download the binary. + destDir := filepath.Join(taskDir, allocdir.TaskLocal) vmID := fmt.Sprintf("qemu-vm-%s-%s", structs.GenerateUUID(), filepath.Base(source)) - fPath := filepath.Join(taskLocal, vmID) - vmPath, err := os.OpenFile(fPath, os.O_CREATE|os.O_WRONLY, 0666) - if err != nil { - return nil, fmt.Errorf("Error opening file to download to: %s", err) - } - - defer vmPath.Close() - defer resp.Body.Close() - - // Copy remote file to local AllocDir for execution - // TODO: a retry of sort if io.Copy fails, for large binaries - _, ioErr := io.Copy(vmPath, resp.Body) - if ioErr != nil { - return nil, fmt.Errorf("Error copying Qemu image from source: %s", ioErr) + vmPath := filepath.Join(destDir, vmID) + if err := getter.GetFile(vmPath, source); err != nil { + return nil, fmt.Errorf("Error downloading source for Java driver: %s", err) } // compute and check checksum if check, ok := task.Config["checksum"]; ok { d.logger.Printf("[DEBUG] Running checksum on (%s)", vmID) hasher := sha256.New() - file, err := os.Open(vmPath.Name()) + file, err := os.Open(vmPath) if err != nil { return nil, fmt.Errorf("Failed to open file for checksum") } @@ -163,7 +143,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, "-machine", "type=pc,accel=" + accelerator, "-name", vmID, "-m", mem, - "-drive", "file=" + vmPath.Name(), + "-drive", "file=" + vmPath, "-nodefconfig", "-nodefaults", "-nographic", @@ -240,7 +220,7 @@ func (d *QemuDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, // Create and Return Handle h := &qemuHandle{ proc: cmd.Process, - vmID: vmPath.Name(), + vmID: vmPath, doneCh: make(chan struct{}), waitCh: make(chan error, 1), } diff --git a/client/driver/raw_exec.go b/client/driver/raw_exec.go index cdc41e676ad6..697e2e4a23ab 100644 --- a/client/driver/raw_exec.go +++ b/client/driver/raw_exec.go @@ -2,14 +2,18 @@ package driver import ( "fmt" + "log" "os" "os/exec" + "path" "path/filepath" "runtime" "strconv" "strings" + "syscall" "time" + "github.com/hashicorp/go-getter" "github.com/hashicorp/nomad/client/allocdir" "github.com/hashicorp/nomad/client/config" "github.com/hashicorp/nomad/client/driver/args" @@ -61,12 +65,6 @@ func (d *RawExecDriver) Fingerprint(cfg *config.Config, node *structs.Node) (boo } func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { - // Get the command - command, ok := task.Config["command"] - if !ok || command == "" { - return nil, fmt.Errorf("missing command for raw_exec driver") - } - // Get the tasks local directory. taskName := d.DriverContext.taskName taskDir, ok := ctx.AllocDir.TaskDirs[taskName] @@ -75,6 +73,40 @@ func (d *RawExecDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandl } taskLocal := filepath.Join(taskDir, allocdir.TaskLocal) + // Get the command to be ran, or if omitted, download an artifact for + // execution. Currently a supplied command takes precedence, and an artifact + // is only downloaded if no command is supplied + command, ok := task.Config["command"] + if !ok || command == "" { + return nil, fmt.Errorf("missing command for exec driver") + } + + source, ok := task.Config["artifact_source"] + if ok && source != "" { + // Proceed to download an artifact to be executed. + // We use go-getter to support a variety of protocols, but need to change + // file permissions of the resulted download to be executable + + // Create a location to download the artifact. + destDir := filepath.Join(taskDir, allocdir.TaskLocal) + + artifactName := path.Base(source) + artifactFile := filepath.Join(destDir, artifactName) + if err := getter.GetFile(artifactFile, source); err != nil { + return nil, fmt.Errorf("[Err] driver.Exec: Error downloading source for Exec driver: %s", err) + } + + // Add execution permissions to the newly downloaded artifact + if runtime.GOOS != "windows" { + if err := syscall.Chmod(artifactFile, 0655); err != nil { + log.Printf("[Err] driver.Exec: Error making artifact executable: %s", err) + } + } + + // re-assign the command to be the local execution path + command = filepath.Join(allocdir.TaskLocal, command) + } + // Get the environment variables. envVars := TaskEnvironmentVariables(ctx, task) diff --git a/client/driver/raw_exec_test.go b/client/driver/raw_exec_test.go index f3b63e12e431..25d1ee696216 100644 --- a/client/driver/raw_exec_test.go +++ b/client/driver/raw_exec_test.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "path/filepath" "reflect" + "runtime" "testing" "time" @@ -92,6 +93,59 @@ func TestRawExecDriver_StartOpen_Wait(t *testing.T) { } } +func TestRawExecDriver_Start_Artifact_Wait(t *testing.T) { + var file string + switch runtime.GOOS { + case "darwin": + file = "hi_darwin_amd64" + default: + file = "hi_linux_amd64" + } + + task := &structs.Task{ + Name: "sleep", + Config: map[string]string{ + "artifact_source": fmt.Sprintf("https://dl.dropboxusercontent.com/u/47675/jar_thing/%s", file), + "command": file, + }, + } + driverCtx := testDriverContext(task.Name) + ctx := testDriverExecContext(task, driverCtx) + defer ctx.AllocDir.Destroy() + + d := NewRawExecDriver(driverCtx) + handle, err := d.Start(ctx, task) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle == nil { + t.Fatalf("missing handle") + } + + // Attempt to open + handle2, err := d.Open(ctx, handle.ID()) + if err != nil { + t.Fatalf("err: %v", err) + } + if handle2 == nil { + t.Fatalf("missing handle") + } + + // Task should terminate quickly + select { + case <-handle2.WaitCh(): + case <-time.After(5 * time.Second): + t.Fatalf("timeout") + } + + // Check they are both tracking the same PID. + pid1 := handle.(*rawExecHandle).proc.Pid + pid2 := handle2.(*rawExecHandle).proc.Pid + if pid1 != pid2 { + t.Fatalf("tracking incorrect Pid; %v != %v", pid1, pid2) + } +} + func TestRawExecDriver_Start_Wait(t *testing.T) { task := &structs.Task{ Name: "sleep", diff --git a/client/executor/exec_universal.go b/client/executor/exec_universal.go index 30e40fd43620..72af769d2290 100644 --- a/client/executor/exec_universal.go +++ b/client/executor/exec_universal.go @@ -29,7 +29,11 @@ func (e *UniversalExecutor) Limit(resources *structs.Resources) error { } func (e *UniversalExecutor) ConfigureTaskDir(taskName string, alloc *allocdir.AllocDir) error { - // No-op + taskDir, ok := alloc.TaskDirs[taskName] + if !ok { + return fmt.Errorf("Error finding task dir for (%s)", taskName) + } + e.Dir = taskDir return nil } diff --git a/website/.ruby-version b/website/.ruby-version new file mode 100644 index 000000000000..b1b25a5ffae4 --- /dev/null +++ b/website/.ruby-version @@ -0,0 +1 @@ +2.2.2 diff --git a/website/source/docs/drivers/exec.html.md b/website/source/docs/drivers/exec.html.md index 1f3e50935c0e..bc18cb107771 100644 --- a/website/source/docs/drivers/exec.html.md +++ b/website/source/docs/drivers/exec.html.md @@ -21,6 +21,8 @@ scripts or other wrappers which provide higher level features. The `exec` driver supports the following configuration in the job spec: * `command` - The command to execute. Must be provided. +* `artifact_source` – Source location of an executable artifact. Must be accessible +from the Nomad client * `args` - The argument list to the command, space seperated. Optional. @@ -30,6 +32,10 @@ The `exec` driver can run on all supported operating systems but to provide proper isolation the client must be run as root on non-Windows operating systems. Further, to support cgroups, `/sys/fs/cgroups/` must be mounted. +You must specify a `command` to be executed. Optionally you can specify an +`artifact_source` to be executed. Any `command` is assumed to be present on the +running client, or a downloaded artifact + ## Client Attributes The `exec` driver will set the following client attributes: diff --git a/website/source/docs/drivers/java.html.md b/website/source/docs/drivers/java.html.md index 62e1636a3adb..f174bad657ec 100644 --- a/website/source/docs/drivers/java.html.md +++ b/website/source/docs/drivers/java.html.md @@ -19,7 +19,7 @@ HTTP from the Nomad client. The `java` driver supports the following configuration in the job spec: * `jar_source` - **(Required)** The hosted location of the source Jar file. Must be accessible -from the Nomad client, via HTTP +from the Nomad client * `args` - **(Optional)** The argument list for the `java` command, space separated. diff --git a/website/source/docs/drivers/raw_exec.html.md b/website/source/docs/drivers/raw_exec.html.md index 35b0c95bf76a..9c51e3cb7d2a 100644 --- a/website/source/docs/drivers/raw_exec.html.md +++ b/website/source/docs/drivers/raw_exec.html.md @@ -19,7 +19,8 @@ As such, it should be used with extreme care and is disabled by default. The `raw_exec` driver supports the following configuration in the job spec: * `command` - The command to execute. Must be provided. - +* `artifact_source` – Source location of an executable artifact. Must be accessible +from the Nomad client * `args` - The argument list to the command, space seperated. Optional. ## Client Requirements @@ -35,6 +36,10 @@ options = { } ``` +You must specify a `command` to be executed. Optionally you can specify an +`artifact_source` to be executed. Any `command` is assumed to be present on the +running client, or a downloaded artifact + ## Client Attributes The `raw_exec` driver will set the following client attributes: