Skip to content

Commit

Permalink
docker: generate /etc/hosts file for bridge network mode
Browse files Browse the repository at this point in the history
When `network.mode = "bridge"`, we create a pause container in Docker with no
networking so that we have a process to hold the network namespace we create
in Nomad. The default `/etc/hosts` file of that pause container is then used
for all the Docker tasks that share that network namespace. Some applications
rely on this file being populated.

This changeset generates a `/etc/hosts` file and bind-mounts it to the
container when Nomad owns the network, so that the container's hostname has an
IP in the file as expected.
  • Loading branch information
tgross committed Jun 16, 2021
1 parent ca010f9 commit 95ff1d7
Show file tree
Hide file tree
Showing 7 changed files with 457 additions and 267 deletions.
7 changes: 6 additions & 1 deletion client/allocrunner/network_hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"github.com/hashicorp/nomad/plugins/drivers"
)

const dockerNetSpecLabelKey = "docker_sandbox_container_id"

type networkIsolationSetter interface {
SetNetworkIsolation(*drivers.NetworkIsolationSpec)
}
Expand Down Expand Up @@ -106,7 +108,10 @@ func (h *networkHook) Prerun() error {
if err != nil {
return fmt.Errorf("failed to configure networking for alloc: %v", err)
}

h.spec.HostsConfig = &drivers.HostsConfig{
Address: status.Address,
Hostname: spec.Labels[dockerNetSpecLabelKey][:12], // TODO: make safe
}
h.networkStatusSetter.SetNetworkStatus(status)
}
return nil
Expand Down
25 changes: 25 additions & 0 deletions drivers/docker/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/hashicorp/nomad/drivers/docker/docklog"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/nomad/drivers/shared/eventer"
"github.com/hashicorp/nomad/drivers/shared/hostnames"
"github.com/hashicorp/nomad/drivers/shared/resolvconf"
nstructs "github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
Expand Down Expand Up @@ -954,6 +955,30 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
hostConfig.Mounts = append(hostConfig.Mounts, *hm)
}

// Setup /etc/hosts
// If the task's network_mode is unset our hostname and IP will come from
// the Nomad-owned network (if in use), so we need to generate an
// /etc/hosts file that matches the network rather than the default one
// that comes from the pause container
if task.NetworkIsolation != nil && driverConfig.NetworkMode == "" {
etcHostMount, err := hostnames.GenerateEtcHostsMount(
task.TaskDir().Dir, task.NetworkIsolation, driverConfig.ExtraHosts)
if err != nil {
return c, fmt.Errorf("failed to build mount for /etc/hosts: %v", err)
}
if etcHostMount != nil {
hostConfig.Mounts = append(hostConfig.Mounts, docker.HostMount{
Target: etcHostMount.TaskPath,
Source: etcHostMount.HostPath,
Type: "bind",
ReadOnly: etcHostMount.Readonly,
BindOptions: &docker.BindOptions{
Propagation: etcHostMount.PropagationMode,
},
})
}
}

// Setup DNS
// If task DNS options are configured Nomad will manage the resolv.conf file
// Docker driver dns options are not compatible with task dns options
Expand Down
64 changes: 64 additions & 0 deletions drivers/shared/hostnames/mount.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package hostnames

import (
"fmt"
"io/ioutil"
"path/filepath"

"github.com/hashicorp/nomad/plugins/drivers"
)

// GenerateEtcHostsMount writes a /etc/hosts file using the network spec's
// hosts configuration, and returns a mount config so that task drivers can
// bind-mount it into the resulting task's filesystem.
func GenerateEtcHostsMount(taskDir string, conf *drivers.NetworkIsolationSpec, extraHosts []string) (*drivers.MountConfig, error) {
if conf == nil || conf.Mode != drivers.NetIsolationModeGroup {
return nil, nil
}
hostsCfg := conf.HostsConfig
if hostsCfg == nil || hostsCfg.Address == "" || hostsCfg.Hostname == "" {
return nil, nil
}

content := fmt.Sprintf(`# this file was generated by Nomad
127.0.0.1 localhost
::1 localhost
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
# this entry is the IP address and hostname of the allocation
# shared with tasks in the task group's network
%s %s
`, hostsCfg.Address, hostsCfg.Hostname)

if len(extraHosts) > 0 {
content += "\n# these entries are extra hosts added by the task config"
for _, hostLine := range extraHosts {
content += "\n" + hostLine
}
content += "\n"
}

path := filepath.Join(taskDir, "hosts")
err := ioutil.WriteFile(path, []byte(content), 0755)
if err != nil {
return nil, err
}

// Note that we're not setting readonly. The file is in the task dir
// anyways, so this lets the task overwrite its own hosts file if the
// application knows better than Nomad here. Task drivers may override
// this behavior.
mount := &drivers.MountConfig{
TaskPath: "/etc/hosts",
HostPath: path,
Readonly: false,
PropagationMode: "private",
}

return mount, nil
}
12 changes: 9 additions & 3 deletions plugins/drivers/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,15 @@ var (
)

type NetworkIsolationSpec struct {
Mode NetIsolationMode
Path string
Labels map[string]string
Mode NetIsolationMode
Path string
Labels map[string]string
HostsConfig *HostsConfig
}

type HostsConfig struct {
Hostname string
Address string
}

// MountConfigSupport is an enum that defaults to "all" for backwards
Expand Down
Loading

0 comments on commit 95ff1d7

Please sign in to comment.