docker: use Nomad managed resolv.conf when DNS options are set (#8600)

drivers/docker/driver.go

@@ -23,6 +23,7 @@ import (
 	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/drivers/docker/docklog"
 	"github.com/hashicorp/nomad/drivers/shared/eventer"
+	"github.com/hashicorp/nomad/drivers/shared/resolvconf"
 	nstructs "github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/base"
 	"github.com/hashicorp/nomad/plugins/drivers"
@@ -920,28 +921,6 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 		hostConfig.ShmSize = driverConfig.ShmSize
 	}
 
-	// setup Nomad DNS options, these are overridden by docker driver specific options
-	if task.DNS != nil {
-		hostConfig.DNS = task.DNS.Servers
-		hostConfig.DNSSearch = task.DNS.Searches
-		hostConfig.DNSOptions = task.DNS.Options
-	}
-
-	if len(driverConfig.DNSSearchDomains) > 0 {
-		hostConfig.DNSSearch = driverConfig.DNSSearchDomains
-	}
-	if len(driverConfig.DNSOptions) > 0 {
-		hostConfig.DNSOptions = driverConfig.DNSOptions
-	}
-	// set DNS servers
-	for _, ip := range driverConfig.DNSServers {
-		if net.ParseIP(ip) != nil {
-			hostConfig.DNS = append(hostConfig.DNS, ip)
-		} else {
-			logger.Error("invalid ip address for container dns server", "ip", ip)
-		}
-	}
-
 	// Setup devices
 	for _, device := range driverConfig.Devices {
 		dd, err := device.toDockerDevice()
@@ -977,6 +956,40 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
 		hostConfig.Mounts = append(hostConfig.Mounts, hm)
 	}
 
+	// Setup DNS
+	// If task DNS options are configured Nomad will manage the resolv.conf file
+	// Docker driver dns options are not compatible with task dns options
+	if task.DNS != nil {
+		dnsMount, err := resolvconf.GenerateDNSMount(task.TaskDir().Dir, task.DNS)
+		if err != nil {
+			return c, fmt.Errorf("failed to build mount for resolv.conf: %v", err)
+		}
+		hostConfig.Mounts = append(hostConfig.Mounts, docker.HostMount{
+			Target:   dnsMount.TaskPath,
+			Source:   dnsMount.HostPath,
+			Type:     "bind",
+			ReadOnly: dnsMount.Readonly,
+			BindOptions: &docker.BindOptions{
+				Propagation: dnsMount.PropagationMode,
+			},
+		})
+	} else {
+		if len(driverConfig.DNSSearchDomains) > 0 {
+			hostConfig.DNSSearch = driverConfig.DNSSearchDomains
+		}
+		if len(driverConfig.DNSOptions) > 0 {
+			hostConfig.DNSOptions = driverConfig.DNSOptions
+		}
+		// set DNS servers
+		for _, ip := range driverConfig.DNSServers {
+			if net.ParseIP(ip) != nil {
+				hostConfig.DNS = append(hostConfig.DNS, ip)
+			} else {
+				logger.Error("invalid ip address for container dns server", "ip", ip)
+			}
+		}
+	}
+
 	for _, m := range task.Mounts {
 		hm := docker.HostMount{
 			Type:     "bind",

drivers/docker/driver_test.go

@@ -1338,25 +1338,46 @@ func TestDockerDriver_DNS(t *testing.T) {
 		t.Parallel()
 	}
 	testutil.DockerCompatible(t)
+	testutil.ExecCompatible(t)
 
-	task, cfg, ports := dockerTask(t)
-	defer freeport.Return(ports)
-	cfg.DNSServers = []string{"8.8.8.8", "8.8.4.4"}
-	cfg.DNSSearchDomains = []string{"example.com", "example.org", "example.net"}
-	cfg.DNSOptions = []string{"ndots:1"}
-	require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
+	cases := []struct {
+		name string
+		cfg  *drivers.DNSConfig
+	}{
+		{
+			name: "nil DNSConfig",
+		},
+		{
+			name: "basic",
+			cfg: &drivers.DNSConfig{
+				Servers: []string{"1.1.1.1", "1.0.0.1"},
+			},
+		},
+		{
+			name: "full",
+			cfg: &drivers.DNSConfig{
+				Servers:  []string{"1.1.1.1", "1.0.0.1"},
+				Searches: []string{"local.test", "node.consul"},
+				Options:  []string{"ndots:2", "edns0"},
+			},
+		},
+	}
 
-	client, d, handle, cleanup := dockerSetup(t, task, nil)
-	defer cleanup()
+	for _, c := range cases {
+		task, cfg, ports := dockerTask(t)
+		defer freeport.Return(ports)
+		task.DNS = c.cfg
+		require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
 
-	require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second))
+		_, d, _, cleanup := dockerSetup(t, task, nil)
+		defer cleanup()
 
-	container, err := client.InspectContainer(handle.containerID)
-	require.NoError(t, err)
+		require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second))
+		defer d.DestroyTask(task.ID, true)
+
+		dtestutil.TestTaskDNSConfig(t, d, task.ID, c.cfg)
+	}
 
-	require.Exactly(t, cfg.DNSServers, container.HostConfig.DNS)
-	require.Exactly(t, cfg.DNSSearchDomains, container.HostConfig.DNSSearch)
-	require.Exactly(t, cfg.DNSOptions, container.HostConfig.DNSOptions)
 }
 
 func TestDockerDriver_MemoryHardLimit(t *testing.T) {