From 151dacde8adc31655e4b8ddf817be57d2dd97725 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 6 Aug 2020 14:18:14 -0400 Subject: [PATCH 1/2] docker: use Nomad managed resolv.conf when DNS options are set --- drivers/docker/driver.go | 57 +++++++++++++++++++++-------------- drivers/docker/driver_test.go | 48 ++++++++++++++++++++--------- 2 files changed, 69 insertions(+), 36 deletions(-) diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index 59cb459198d2..7462db65f274 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/nomad/client/taskenv" "github.com/hashicorp/nomad/drivers/docker/docklog" "github.com/hashicorp/nomad/drivers/shared/eventer" + "github.com/hashicorp/nomad/drivers/shared/resolvconf" nstructs "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/plugins/base" "github.com/hashicorp/nomad/plugins/drivers" @@ -915,28 +916,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.ShmSize = driverConfig.ShmSize } - // setup Nomad DNS options, these are overridden by docker driver specific options - if task.DNS != nil { - hostConfig.DNS = task.DNS.Servers - hostConfig.DNSSearch = task.DNS.Searches - hostConfig.DNSOptions = task.DNS.Options - } - - if len(driverConfig.DNSSearchDomains) > 0 { - hostConfig.DNSSearch = driverConfig.DNSSearchDomains - } - if len(driverConfig.DNSOptions) > 0 { - hostConfig.DNSOptions = driverConfig.DNSOptions - } - // set DNS servers - for _, ip := range driverConfig.DNSServers { - if net.ParseIP(ip) != nil { - hostConfig.DNS = append(hostConfig.DNS, ip) - } else { - logger.Error("invalid ip address for container dns server", "ip", ip) - } - } - // Setup devices for _, device := range driverConfig.Devices { dd, err := device.toDockerDevice() @@ -972,6 +951,40 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T hostConfig.Mounts = append(hostConfig.Mounts, hm) } + // Setup DNS + // If task DNS options are configured Nomad will manage the resolv.conf file + // Docker driver dns options are not compatible with task dns options + if task.DNS != nil { + dnsMount, err := resolvconf.GenerateDNSMount(task.TaskDir().Dir, task.DNS) + if err != nil { + return c, fmt.Errorf("failed to build mount for resolv.conf: %v", err) + } + hostConfig.Mounts = append(hostConfig.Mounts, docker.HostMount{ + Target: dnsMount.TaskPath, + Source: dnsMount.HostPath, + Type: "bind", + ReadOnly: dnsMount.Readonly, + BindOptions: &docker.BindOptions{ + Propagation: dnsMount.PropagationMode, + }, + }) + } else { + if len(driverConfig.DNSSearchDomains) > 0 { + hostConfig.DNSSearch = driverConfig.DNSSearchDomains + } + if len(driverConfig.DNSOptions) > 0 { + hostConfig.DNSOptions = driverConfig.DNSOptions + } + // set DNS servers + for _, ip := range driverConfig.DNSServers { + if net.ParseIP(ip) != nil { + hostConfig.DNS = append(hostConfig.DNS, ip) + } else { + logger.Error("invalid ip address for container dns server", "ip", ip) + } + } + } + for _, m := range task.Mounts { hm := docker.HostMount{ Type: "bind", diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index 9ba0e8e4a0fc..c5cde3522d68 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -1338,24 +1338,44 @@ func TestDockerDriver_DNS(t *testing.T) { } testutil.DockerCompatible(t) - task, cfg, ports := dockerTask(t) - defer freeport.Return(ports) - cfg.DNSServers = []string{"8.8.8.8", "8.8.4.4"} - cfg.DNSSearchDomains = []string{"example.com", "example.org", "example.net"} - cfg.DNSOptions = []string{"ndots:1"} - require.NoError(t, task.EncodeConcreteDriverConfig(cfg)) + cases := []struct { + name string + cfg *drivers.DNSConfig + }{ + { + name: "nil DNSConfig", + }, + { + name: "basic", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + }, + }, + { + name: "full", + cfg: &drivers.DNSConfig{ + Servers: []string{"1.1.1.1", "1.0.0.1"}, + Searches: []string{"local.test", "node.consul"}, + Options: []string{"ndots:2", "edns0"}, + }, + }, + } - client, d, handle, cleanup := dockerSetup(t, task, nil) - defer cleanup() + for _, c := range cases { + task, cfg, ports := dockerTask(t) + defer freeport.Return(ports) + task.DNS = c.cfg + require.NoError(t, task.EncodeConcreteDriverConfig(cfg)) - require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second)) + _, d, _, cleanup := dockerSetup(t, task, nil) + defer cleanup() - container, err := client.InspectContainer(handle.containerID) - require.NoError(t, err) + require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second)) + defer d.DestroyTask(task.ID, true) + + dtestutil.TestTaskDNSConfig(t, d, task.ID, c.cfg) + } - require.Exactly(t, cfg.DNSServers, container.HostConfig.DNS) - require.Exactly(t, cfg.DNSSearchDomains, container.HostConfig.DNSSearch) - require.Exactly(t, cfg.DNSOptions, container.HostConfig.DNSOptions) } func TestDockerDriver_MemoryHardLimit(t *testing.T) { From 04ee9896ea438a718258a5a54bc99fc6b11d86f1 Mon Sep 17 00:00:00 2001 From: Nick Ethier Date: Thu, 6 Aug 2020 15:33:38 -0400 Subject: [PATCH 2/2] docker: skip windows for dns test --- drivers/docker/driver_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/docker/driver_test.go b/drivers/docker/driver_test.go index c5cde3522d68..dd70e0137573 100644 --- a/drivers/docker/driver_test.go +++ b/drivers/docker/driver_test.go @@ -1337,6 +1337,7 @@ func TestDockerDriver_DNS(t *testing.T) { t.Parallel() } testutil.DockerCompatible(t) + testutil.ExecCompatible(t) cases := []struct { name string