allow configuration of Docker hostnames in bridge mode (#11173)

Add a new hostname string parameter to the network block which
allows operators to specify the hostname of the network namespace.
Changing this causes a destructive update to the allocation and it
is omitted if empty from API responses. This parameter also supports
interpolation.

In order to have a hostname passed as a configuration param when
creating an allocation network, the CreateNetwork func of the
DriverNetworkManager interface needs to be updated. In order to
minimize the disruption of future changes, rather than add another
string func arg, the function now accepts a request struct along with
the allocID param. The struct has the hostname as a field.

The in-tree implementations of DriverNetworkManager.CreateNetwork
have been modified to account for the function signature change.
In updating for the change, the enhancement of adding hostnames to
network namespaces has also been added to the Docker driver, whilst
the default Linux manager does not current implement it.

api/resources.go

@@ -119,6 +119,7 @@ type NetworkResource struct {
 	DNS           *DNSConfig `hcl:"dns,block"`
 	ReservedPorts []Port     `hcl:"reserved_ports,block"`
 	DynamicPorts  []Port     `hcl:"port,block"`
+	Hostname      string     `hcl:"hostname,optional"`
 
 	// COMPAT(0.13)
 	// XXX Deprecated. Please do not use. The field will be removed in Nomad

client/allocrunner/alloc_runner_hooks.go

@@ -133,6 +133,14 @@ func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error {
 		return fmt.Errorf("failed to initialize network configurator: %v", err)
 	}
 
+	// Create a new taskenv.Builder which is used and mutated by networkHook.
+	envBuilder := taskenv.NewBuilder(
+		config.Node, ar.Alloc(), nil, config.Region).SetAllocDir(ar.allocDir.AllocDir)
+
+	// Create a taskenv.TaskEnv which is used for read only purposes by the
+	// newNetworkHook.
+	builtTaskEnv := envBuilder.Build()
+
 	// Create the alloc directory hook. This is run first to ensure the
 	// directory path exists for other hooks.
 	alloc := ar.Alloc()
@@ -142,13 +150,13 @@ func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error {
 		newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher),
 		newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir),
 		newAllocHealthWatcherHook(hookLogger, alloc, hs, ar.Listener(), ar.consulClient),
-		newNetworkHook(hookLogger, ns, alloc, nm, nc, ar),
+		newNetworkHook(hookLogger, ns, alloc, nm, nc, ar, builtTaskEnv),
 		newGroupServiceHook(groupServiceHookConfig{
 			alloc:               alloc,
 			consul:              ar.consulClient,
 			consulNamespace:     alloc.ConsulNamespace(),
 			restarter:           ar,
-			taskEnvBuilder:      taskenv.NewBuilder(config.Node, ar.Alloc(), nil, config.Region).SetAllocDir(ar.allocDir.AllocDir),
+			taskEnvBuilder:      envBuilder,
 			networkStatusGetter: ar,
 			logger:              hookLogger,
 		}),

client/allocrunner/network_hook.go

@@ -5,14 +5,25 @@ import (
 	"fmt"
 
 	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/hashicorp/nomad/plugins/drivers"
+	"github.com/miekg/dns"
 )
 
-// We create a pause container to own the network namespace, and the
-// NetworkIsolationSpec we get back from CreateNetwork has this label set as
-// the container ID. We'll use this to generate a hostname for the task.
-const dockerNetSpecLabelKey = "docker_sandbox_container_id"
+const (
+	// dockerNetSpecLabelKey is the label added when we create a pause
+	// container to own the network namespace, and the NetworkIsolationSpec we
+	// get back from CreateNetwork has this label set as the container ID.
+	// We'll use this to generate a hostname for the task in the event the user
+	// did not specify a custom one. Please see dockerNetSpecHostnameKey.
+	dockerNetSpecLabelKey = "docker_sandbox_container_id"
+
+	// dockerNetSpecHostnameKey is the label added when we create a pause
+	// container and the task group network include a user supplied hostname
+	// parameter.
+	dockerNetSpecHostnameKey = "docker_sandbox_hostname"
+)
 
 type networkIsolationSetter interface {
 	SetNetworkIsolation(*drivers.NetworkIsolationSpec)
@@ -61,6 +72,9 @@ type networkHook struct {
 	// the alloc network has been created
 	networkConfigurator NetworkConfigurator
 
+	// taskEnv is used to perform interpolation within the network blocks.
+	taskEnv *taskenv.TaskEnv
+
 	logger hclog.Logger
 }
 
@@ -69,13 +83,16 @@ func newNetworkHook(logger hclog.Logger,
 	alloc *structs.Allocation,
 	netManager drivers.DriverNetworkManager,
 	netConfigurator NetworkConfigurator,
-	networkStatusSetter networkStatusSetter) *networkHook {
+	networkStatusSetter networkStatusSetter,
+	taskEnv *taskenv.TaskEnv,
+) *networkHook {
 	return &networkHook{
 		isolationSetter:     ns,
 		networkStatusSetter: networkStatusSetter,
 		alloc:               alloc,
 		manager:             netManager,
 		networkConfigurator: netConfigurator,
+		taskEnv:             taskEnv,
 		logger:              logger,
 	}
 }
@@ -95,8 +112,24 @@ func (h *networkHook) Prerun() error {
 		return nil
 	}
 
-	spec, created, err := h.manager.CreateNetwork(h.alloc.ID)
+	// Perform our networks block interpolation.
+	interpolatedNetworks := taskenv.InterpolateNetworks(h.taskEnv, tg.Networks)
+
+	// Interpolated values need to be validated. It is also possible a user
+	// supplied hostname avoids the validation on job registrations because it
+	// looks like it includes interpolation, when it doesn't.
+	if interpolatedNetworks[0].Hostname != "" {
+		if _, ok := dns.IsDomainName(interpolatedNetworks[0].Hostname); !ok {
+			return fmt.Errorf("network hostname %q is not a valid DNS name", interpolatedNetworks[0].Hostname)
+		}
+	}
+
+	// Our network create request.
+	networkCreateReq := drivers.NetworkCreateRequest{
+		Hostname: interpolatedNetworks[0].Hostname,
+	}
 
+	spec, created, err := h.manager.CreateNetwork(h.alloc.ID, &networkCreateReq)
 	if err != nil {
 		return fmt.Errorf("failed to create network for alloc: %v", err)
 	}
@@ -111,18 +144,31 @@ func (h *networkHook) Prerun() error {
 		if err != nil {
 			return fmt.Errorf("failed to configure networking for alloc: %v", err)
 		}
-		if hostname, ok := spec.Labels[dockerNetSpecLabelKey]; ok {
+
+		// If the driver set the sandbox hostname label, then we will use that
+		// to set the HostsConfig.Hostname. Otherwise, identify the sandbox
+		// container ID which will have been used to set the network namespace
+		// hostname.
+		if hostname, ok := spec.Labels[dockerNetSpecHostnameKey]; ok {
+			h.spec.HostsConfig = &drivers.HostsConfig{
+				Address:  status.Address,
+				Hostname: hostname,
+			}
+		} else if hostname, ok := spec.Labels[dockerNetSpecLabelKey]; ok {
+
+			// the docker_sandbox_container_id is the full ID of the pause
+			// container, whereas we want the shortened name that dockerd sets
+			// as the pause container's hostname.
 			if len(hostname) > 12 {
-				// the docker_sandbox_container_id is the full ID of the pause
-				// container, whereas we want the shortened name that dockerd
-				// sets as the pause container's hostname
 				hostname = hostname[:12]
 			}
+
 			h.spec.HostsConfig = &drivers.HostsConfig{
 				Address:  status.Address,
 				Hostname: hostname,
 			}
 		}
+
 		h.networkStatusSetter.SetNetworkStatus(status)
 	}
 	return nil

client/allocrunner/network_hook_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	"github.com/hashicorp/nomad/client/taskenv"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/nomad/mock"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -56,7 +57,7 @@ func TestNetworkHook_Prerun_Postrun(t *testing.T) {
 	destroyCalled := false
 	nm := &testutils.MockDriver{
 		MockNetworkManager: testutils.MockNetworkManager{
-			CreateNetworkF: func(allocID string) (*drivers.NetworkIsolationSpec, bool, error) {
+			CreateNetworkF: func(allocID string, req *drivers.NetworkCreateRequest) (*drivers.NetworkIsolationSpec, bool, error) {
 				require.Equal(t, alloc.ID, allocID)
 				return spec, false, nil
 			},
@@ -79,8 +80,10 @@ func TestNetworkHook_Prerun_Postrun(t *testing.T) {
 	}
 	require := require.New(t)
 
+	envBuilder := taskenv.NewBuilder(mock.Node(), alloc, nil, alloc.Job.Region)
+
 	logger := testlog.HCLogger(t)
-	hook := newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{}, statusSetter)
+	hook := newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{}, statusSetter, envBuilder.Build())
 	require.NoError(hook.Prerun())
 	require.True(setter.called)
 	require.False(destroyCalled)
@@ -91,11 +94,10 @@ func TestNetworkHook_Prerun_Postrun(t *testing.T) {
 	setter.called = false
 	destroyCalled = false
 	alloc.Job.TaskGroups[0].Networks[0].Mode = "host"
-	hook = newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{}, statusSetter)
+	hook = newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{}, statusSetter, envBuilder.Build())
 	require.NoError(hook.Prerun())
 	require.False(setter.called)
 	require.False(destroyCalled)
 	require.NoError(hook.Postrun())
 	require.False(destroyCalled)
-
 }

client/allocrunner/network_manager_linux.go

@@ -26,8 +26,18 @@ func newNetworkManager(alloc *structs.Allocation, driverManager drivermanager.Ma
 		tgNetMode = tg.Networks[0].Mode
 	}
 
+	groupIsolationMode := netModeToIsolationMode(tgNetMode)
+
+	// Setting the hostname is only possible where the task groups networking
+	// mode is group; meaning bridge or none.
+	if len(tg.Networks) > 0 &&
+		(groupIsolationMode != drivers.NetIsolationModeGroup && tg.Networks[0].Hostname != "") {
+		return nil, fmt.Errorf("hostname cannot be set on task group using %q networking mode",
+			groupIsolationMode)
+	}
+
 	// networkInitiator tracks the task driver which needs to create the network
-	// to check for multiple drivers needing the create the network
+	// to check for multiple drivers needing to create the network.
 	var networkInitiator string
 
 	// driverCaps tracks which drivers we've checked capabilities for so as not
@@ -80,6 +90,14 @@ func newNetworkManager(alloc *structs.Allocation, driverManager drivermanager.Ma
 
 			nm = netManager
 			networkInitiator = task.Name
+		} else if tg.Networks[0].Hostname != "" {
+			// TODO jrasell: remove once the default linux network manager
+			//  supports setting the hostname in bridged mode. This currently
+			//  indicates only Docker supports this, which is true unless a
+			//  custom driver can which means this check still holds as true as
+			//  we can tell.
+			//  Please see: https://github.com/hashicorp/nomad/issues/11180
+			return nil, fmt.Errorf("hostname is not currently supported on driver %s", task.Driver)
 		}
 
 		// mark this driver's capabilities as checked
@@ -92,7 +110,10 @@ func newNetworkManager(alloc *structs.Allocation, driverManager drivermanager.Ma
 // defaultNetworkManager creates a network namespace for the alloc
 type defaultNetworkManager struct{}
 
-func (*defaultNetworkManager) CreateNetwork(allocID string) (*drivers.NetworkIsolationSpec, bool, error) {
+// CreateNetwork is the CreateNetwork implementation of the
+// drivers.DriverNetworkManager interface function. It does not currently
+// support setting the hostname of the network namespace.
+func (*defaultNetworkManager) CreateNetwork(allocID string, _ *drivers.NetworkCreateRequest) (*drivers.NetworkIsolationSpec, bool, error) {
 	netns, err := nsutil.NewNS(allocID)
 	if err != nil {
 		// when a client restarts, the namespace will already exist and

client/allocrunner/network_manager_linux_test.go

@@ -164,6 +164,92 @@ func TestNewNetworkManager(t *testing.T) {
 			err:         true,
 			errContains: "want to initiate networking but only one",
 		},
+		{
+			name: "hostname set in bridged mode",
+			alloc: &structs.Allocation{
+				TaskGroup: "group",
+				Job: &structs.Job{
+					TaskGroups: []*structs.TaskGroup{
+						{
+							Name: "group",
+							Networks: []*structs.NetworkResource{
+								{
+									Mode:     "bridge",
+									Hostname: "foobar",
+								},
+							},
+							Tasks: []*structs.Task{
+								{
+									Name:      "task1",
+									Driver:    "mustinit1",
+									Resources: &structs.Resources{},
+								},
+							},
+						},
+					},
+				},
+			},
+			mustInit: true,
+			err:      false,
+		},
+		{
+			name: "hostname set in host mode",
+			alloc: &structs.Allocation{
+				TaskGroup: "group",
+				Job: &structs.Job{
+					TaskGroups: []*structs.TaskGroup{
+						{
+							Name: "group",
+							Networks: []*structs.NetworkResource{
+								{
+									Mode:     "host",
+									Hostname: "foobar",
+								},
+							},
+							Tasks: []*structs.Task{
+								{
+									Name:      "task1",
+									Driver:    "group1",
+									Resources: &structs.Resources{},
+								},
+							},
+						},
+					},
+				},
+			},
+			mustInit:    false,
+			err:         true,
+			errContains: `hostname cannot be set on task group using "host" networking mode`,
+		},
+		{
+			name: "hostname set using exec driver",
+			alloc: &structs.Allocation{
+				TaskGroup: "group",
+				Job: &structs.Job{
+					TaskGroups: []*structs.TaskGroup{
+						{
+							Name: "group",
+							Networks: []*structs.NetworkResource{
+								{
+									Mode:     "bridge",
+									Hostname: "foobar",
+								},
+							},
+							Tasks: []*structs.Task{
+								{
+									Name:      "task1",
+									Driver:    "group1",
+									Resources: &structs.Resources{},
+								},
+							},
+						},
+					},
+				},
+			},
+			mustInit:    false,
+			err:         true,
+			errContains: "hostname is not currently supported on driver group1",
+		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			require := require.New(t)

client/taskenv/network.go

@@ -0,0 +1,29 @@
+package taskenv
+
+import (
+	"github.com/hashicorp/nomad/nomad/structs"
+)
+
+// InterpolateNetworks returns an interpolated copy of the task group networks
+// with values from the task's environment.
+//
+// Current interoperable fields:
+//   - Hostname
+func InterpolateNetworks(taskEnv *TaskEnv, networks structs.Networks) structs.Networks {
+
+	// Guard against not having a valid taskEnv. This can be the case if the
+	// PreKilling or Exited hook is run before Poststart.
+	if taskEnv == nil || networks == nil {
+		return nil
+	}
+
+	// Create a copy of the networks array, so we can manipulate the copy.
+	interpolated := networks.Copy()
+
+	// Iterate the copy and perform the interpolation.
+	for i := range interpolated {
+		interpolated[i].Hostname = taskEnv.ReplaceEnv(interpolated[i].Hostname)
+	}
+
+	return interpolated
+}