From 9bb2088ff5394711aa8561302278701ba4d69a37 Mon Sep 17 00:00:00 2001 From: Michael Schurter Date: Mon, 5 Nov 2018 17:39:02 -0800 Subject: [PATCH] client: interpolate driver configurations Also add missing SetDriverNetwork calls. --- client/allocrunner/taskrunner/task_runner.go | 2 +- .../taskrunner/task_runner_getters.go | 6 +- .../taskrunner/task_runner_test.go | 59 ++++++++++++ client/config/testing.go | 8 +- client/driver/env/env.go | 15 +++ client/driver/env/env_test.go | 95 +++++++++++++++++++ drivers/mock/driver.go | 25 +++++ 7 files changed, 202 insertions(+), 8 deletions(-) diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go index b90f45457b71..699245eb5b99 100644 --- a/client/allocrunner/taskrunner/task_runner.go +++ b/client/allocrunner/taskrunner/task_runner.go @@ -462,8 +462,8 @@ func (tr *TaskRunner) runDriver() error { // TODO(nickethier): make sure this uses alloc.AllocatedResources once #4750 is rebased taskConfig := tr.buildTaskConfig() - // TODO: load variables evalCtx := &hcl.EvalContext{ + Variables: tr.envBuilder.Build().AllValues(), Functions: shared.GetStdlibFuncs(), } diff --git a/client/allocrunner/taskrunner/task_runner_getters.go b/client/allocrunner/taskrunner/task_runner_getters.go index 3bce6ffcd77d..89d67a99ebb0 100644 --- a/client/allocrunner/taskrunner/task_runner_getters.go +++ b/client/allocrunner/taskrunner/task_runner_getters.go @@ -61,11 +61,15 @@ func (tr *TaskRunner) getDriverHandle() *DriverHandle { return tr.handle } -// setDriverHanlde sets the driver handle and creates a new result proxy. +// setDriverHanlde sets the driver handle, creates a new result proxy, and +// updates the driver network in the task's environment. func (tr *TaskRunner) setDriverHandle(handle *DriverHandle) { tr.handleLock.Lock() defer tr.handleLock.Unlock() tr.handle = handle + + // Update the environment's driver network + tr.envBuilder.SetDriverNetwork(handle.net) } func (tr *TaskRunner) clearDriverHandle() { diff --git a/client/allocrunner/taskrunner/task_runner_test.go b/client/allocrunner/taskrunner/task_runner_test.go index f7da0f7e3688..ce75341653a7 100644 --- a/client/allocrunner/taskrunner/task_runner_test.go +++ b/client/allocrunner/taskrunner/task_runner_test.go @@ -12,6 +12,7 @@ import ( consulapi "github.com/hashicorp/nomad/client/consul" cstate "github.com/hashicorp/nomad/client/state" "github.com/hashicorp/nomad/client/vaultclient" + mockdriver "github.com/hashicorp/nomad/drivers/mock" "github.com/hashicorp/nomad/helper/testlog" "github.com/hashicorp/nomad/nomad/mock" "github.com/hashicorp/nomad/nomad/structs" @@ -155,3 +156,61 @@ func TestTaskRunner_Restore_Running(t *testing.T) { } assert.Equal(t, 1, started) } + +// TestTaskRunner_TaskEnv asserts driver configurations are interpolated. +func TestTaskRunner_TaskEnv(t *testing.T) { + t.Parallel() + require := require.New(t) + + alloc := mock.BatchAlloc() + alloc.Job.TaskGroups[0].Meta = map[string]string{ + "common_user": "somebody", + } + task := alloc.Job.TaskGroups[0].Tasks[0] + task.Name = "testtask_taskenv" + task.Driver = "mock_driver" + task.Meta = map[string]string{ + "foo": "bar", + } + + // Use interpolation from both node attributes and meta vars + task.Config = map[string]interface{}{ + "run_for": time.Millisecond, + "stdout_string": `${node.region} ${NOMAD_META_foo}`, + } + task.User = "${NOMAD_META_common_user}" + + conf, cleanup := testTaskRunnerConfig(t, alloc, task.Name) + defer cleanup() + + fmt.Println(conf.ClientConfig.Node) + + // Run the first TaskRunner + tr, err := NewTaskRunner(conf) + require.NoError(err) + go tr.Run() + defer tr.Kill(context.Background(), structs.NewTaskEvent("cleanup")) + + // Wait for task to complete + select { + case <-tr.WaitCh(): + case <-time.After(3 * time.Second): + } + + // Get the mock driver plugin + driverPlugin, err := conf.PluginSingletonLoader.Dispense( + mockdriver.PluginID.Name, + mockdriver.PluginID.PluginType, + nil, + conf.Logger, + ) + require.NoError(err) + mockDriver := driverPlugin.Plugin().(*mockdriver.Driver) + + // Assert its config has been properly interpolated + driverCfg, mockCfg := mockDriver.GetTaskConfig() + require.NotNil(driverCfg) + require.NotNil(mockCfg) + assert.Equal(t, "somebody", driverCfg.User) + assert.Equal(t, "global bar", mockCfg.StdoutString) +} diff --git a/client/config/testing.go b/client/config/testing.go index 73ab82d2d463..1097e5bcde48 100644 --- a/client/config/testing.go +++ b/client/config/testing.go @@ -7,7 +7,7 @@ import ( "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/helper/testlog" - "github.com/hashicorp/nomad/nomad/structs" + "github.com/hashicorp/nomad/nomad/mock" "github.com/mitchellh/go-testing-interface" ) @@ -15,6 +15,7 @@ import ( // a cleanup func to remove the state and alloc dirs when finished. func TestClientConfig(t testing.T) (*Config, func()) { conf := DefaultConfig() + conf.Node = mock.Node() conf.Logger = testlog.HCLogger(t) // Create a tempdir to hold state and alloc subdirs @@ -42,11 +43,6 @@ func TestClientConfig(t testing.T) (*Config, func()) { conf.VaultConfig.Enabled = helper.BoolToPtr(false) conf.DevMode = true - conf.Node = &structs.Node{ - Reserved: &structs.Resources{ - DiskMB: 0, - }, - } // Loosen GC threshold conf.GCDiskUsageThreshold = 98.0 diff --git a/client/driver/env/env.go b/client/driver/env/env.go index 013623168d38..a7872e6e7d1e 100644 --- a/client/driver/env/env.go +++ b/client/driver/env/env.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/helper" hargs "github.com/hashicorp/nomad/helper/args" "github.com/hashicorp/nomad/nomad/structs" + "github.com/zclconf/go-cty/cty" ) // A set of environment variables that are exported by each driver. @@ -159,6 +160,20 @@ func (t *TaskEnv) All() map[string]string { return m } +// AllValues is a map of the task's environment variables and the node's +// attributes with cty.Value (String) values. +func (t *TaskEnv) AllValues() map[string]cty.Value { + m := make(map[string]cty.Value, len(t.EnvMap)+len(t.NodeAttrs)) + for k, v := range t.EnvMap { + m[k] = cty.StringVal(v) + } + for k, v := range t.NodeAttrs { + m[k] = cty.StringVal(v) + } + + return m +} + // ParseAndReplace takes the user supplied args replaces any instance of an // environment variable or Nomad variable in the args with the actual value. func (t *TaskEnv) ParseAndReplace(args []string) []string { diff --git a/client/driver/env/env_test.go b/client/driver/env/env_test.go index db696cc218ad..51e9400d115a 100644 --- a/client/driver/env/env_test.go +++ b/client/driver/env/env_test.go @@ -318,6 +318,101 @@ func TestEnvironment_AsList_Old(t *testing.T) { require.Equal(t, exp, act) } +func TestEnvironment_AllValues(t *testing.T) { + n := mock.Node() + n.Meta = map[string]string{ + "metaKey": "metaVal", + } + a := mock.Alloc() + a.AllocatedResources.Tasks["web"].Networks[0] = &structs.NetworkResource{ + Device: "eth0", + IP: "127.0.0.1", + ReservedPorts: []structs.Port{{Label: "https", Value: 8080}}, + MBits: 50, + DynamicPorts: []structs.Port{{Label: "http", Value: 80}}, + } + a.AllocatedResources.Tasks["ssh"] = &structs.AllocatedTaskResources{ + Networks: []*structs.NetworkResource{ + { + Device: "eth0", + IP: "192.168.0.100", + MBits: 50, + ReservedPorts: []structs.Port{ + {Label: "ssh", Value: 22}, + {Label: "other", Value: 1234}, + }, + }, + }, + } + task := a.Job.TaskGroups[0].Tasks[0] + task.Env = map[string]string{ + "taskEnvKey": "taskEnvVal", + } + env := NewBuilder(n, a, task, "global").SetDriverNetwork( + &cstructs.DriverNetwork{PortMap: map[string]int{"https": 443}}, + ) + + act := env.Build().AllValues() + exp := map[string]string{ + // Node + "node.unique.id": n.ID, + "node.region": "global", + "node.datacenter": n.Datacenter, + "node.unique.name": n.Name, + "node.class": n.NodeClass, + "meta.metaKey": "metaVal", + "attr.arch": "x86", + "attr.driver.exec": "1", + "attr.driver.mock_driver": "1", + "attr.kernel.name": "linux", + "attr.nomad.version": "0.5.0", + + // Env + "taskEnvKey": "taskEnvVal", + "NOMAD_ADDR_http": "127.0.0.1:80", + "NOMAD_PORT_http": "80", + "NOMAD_IP_http": "127.0.0.1", + "NOMAD_ADDR_https": "127.0.0.1:8080", + "NOMAD_PORT_https": "443", + "NOMAD_IP_https": "127.0.0.1", + "NOMAD_HOST_PORT_http": "80", + "NOMAD_HOST_PORT_https": "8080", + "NOMAD_TASK_NAME": "web", + "NOMAD_GROUP_NAME": "web", + "NOMAD_ADDR_ssh_other": "192.168.0.100:1234", + "NOMAD_ADDR_ssh_ssh": "192.168.0.100:22", + "NOMAD_IP_ssh_other": "192.168.0.100", + "NOMAD_IP_ssh_ssh": "192.168.0.100", + "NOMAD_PORT_ssh_other": "1234", + "NOMAD_PORT_ssh_ssh": "22", + "NOMAD_CPU_LIMIT": "500", + "NOMAD_DC": "dc1", + "NOMAD_REGION": "global", + "NOMAD_MEMORY_LIMIT": "256", + "NOMAD_META_ELB_CHECK_INTERVAL": "30s", + "NOMAD_META_ELB_CHECK_MIN": "3", + "NOMAD_META_ELB_CHECK_TYPE": "http", + "NOMAD_META_FOO": "bar", + "NOMAD_META_OWNER": "armon", + "NOMAD_META_elb_check_interval": "30s", + "NOMAD_META_elb_check_min": "3", + "NOMAD_META_elb_check_type": "http", + "NOMAD_META_foo": "bar", + "NOMAD_META_owner": "armon", + "NOMAD_JOB_NAME": "my-job", + "NOMAD_ALLOC_ID": a.ID, + "NOMAD_ALLOC_INDEX": "0", + } + + // Should be able to convert all values back to strings + actStr := make(map[string]string, len(act)) + for k, v := range act { + actStr[k] = v.AsString() + } + + require.Equal(t, exp, actStr) +} + func TestEnvironment_VaultToken(t *testing.T) { n := mock.Node() a := mock.Alloc() diff --git a/drivers/mock/driver.go b/drivers/mock/driver.go index 734ae61f8da3..76b379027432 100644 --- a/drivers/mock/driver.go +++ b/drivers/mock/driver.go @@ -6,6 +6,7 @@ import ( "fmt" "strconv" "strings" + "sync" "time" hclog "github.com/hashicorp/go-hclog" @@ -110,6 +111,15 @@ type Driver struct { shutdownFingerprintTime time.Time + // lastDriverTaskConfig is the last *drivers.TaskConfig passed to StartTask + lastDriverTaskConfig *drivers.TaskConfig + + // lastTaskConfig is the last decoded *TaskConfig created by StartTask + lastTaskConfig *TaskConfig + + // lastMu guards access to last[Driver]TaskConfig + lastMu sync.Mutex + // logger will log to the Nomad agent logger hclog.Logger } @@ -298,6 +308,12 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *cstru return nil, nil, err } + // Store last configs + d.lastMu.Lock() + d.lastDriverTaskConfig = cfg + d.lastTaskConfig = &driverConfig + d.lastMu.Unlock() + if driverConfig.StartBlockFor != 0 { time.Sleep(driverConfig.StartBlockFor) } @@ -455,3 +471,12 @@ func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (* } return &res, nil } + +// GetTaskConfig is unique to the mock driver and for testing purposes only. It +// returns the *drivers.TaskConfig passed to StartTask and the decoded +// *mock.TaskConfig created by the last StartTask call. +func (d *Driver) GetTaskConfig() (*drivers.TaskConfig, *TaskConfig) { + d.lastMu.Lock() + defer d.lastMu.Unlock() + return d.lastDriverTaskConfig, d.lastTaskConfig +}