Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

docker: generate /etc/hosts file for bridge network mode #10766

Merged
merged 1 commit into from
Jun 16, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

IMPROVEMENTS:
* cli: Added `-monitor` flag to `deployment status` command and automatically monitor deployments from `job run` command. [[GH-10661](https://github.com/hashicorp/nomad/pull/10661)]
* docker: Tasks using `network.mode = "bridge"` that don't set their `network_mode` will receive a `/etc/hosts` file that includes the pause container's hostname and any `extra_hosts`. [[GH-10766](https://github.com/hashicorp/nomad/issues/10766)]

BUG FIXES:
* quotas (Enterprise): Fixed a bug where quotas were evaluated before constraints, resulting in quota capacity being used up by filtered nodes. [[GH-10753](https://github.com/hashicorp/nomad/issues/10753)]
Expand Down
18 changes: 17 additions & 1 deletion client/allocrunner/network_hook.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ import (
"github.com/hashicorp/nomad/plugins/drivers"
)

// We create a pause container to own the network namespace, and the
// NetworkIsolationSpec we get back from CreateNetwork has this label set as
// the container ID. We'll use this to generate a hostname for the task.
const dockerNetSpecLabelKey = "docker_sandbox_container_id"
tgross marked this conversation as resolved.
Show resolved Hide resolved

type networkIsolationSetter interface {
SetNetworkIsolation(*drivers.NetworkIsolationSpec)
}
Expand Down Expand Up @@ -106,7 +111,18 @@ func (h *networkHook) Prerun() error {
if err != nil {
return fmt.Errorf("failed to configure networking for alloc: %v", err)
}

if hostname, ok := spec.Labels[dockerNetSpecLabelKey]; ok {
if len(hostname) > 12 {
// the docker_sandbox_container_id is the full ID of the pause
// container, whereas we want the shortened name that dockerd
// sets as the pause container's hostname
hostname = hostname[:12]
}
h.spec.HostsConfig = &drivers.HostsConfig{
Address: status.Address,
Hostname: hostname,
}
}
h.networkStatusSetter.SetNetworkStatus(status)
}
return nil
Expand Down
28 changes: 28 additions & 0 deletions drivers/docker/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/hashicorp/nomad/drivers/docker/docklog"
"github.com/hashicorp/nomad/drivers/shared/capabilities"
"github.com/hashicorp/nomad/drivers/shared/eventer"
"github.com/hashicorp/nomad/drivers/shared/hostnames"
"github.com/hashicorp/nomad/drivers/shared/resolvconf"
nstructs "github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/plugins/base"
Expand Down Expand Up @@ -954,6 +955,33 @@ func (d *Driver) createContainerConfig(task *drivers.TaskConfig, driverConfig *T
hostConfig.Mounts = append(hostConfig.Mounts, *hm)
}

// Setup /etc/hosts
// If the task's network_mode is unset our hostname and IP will come from
// the Nomad-owned network (if in use), so we need to generate an
// /etc/hosts file that matches the network rather than the default one
// that comes from the pause container
if task.NetworkIsolation != nil && driverConfig.NetworkMode == "" {
etcHostMount, err := hostnames.GenerateEtcHostsMount(
task.TaskDir().Dir, task.NetworkIsolation, driverConfig.ExtraHosts)
if err != nil {
return c, fmt.Errorf("failed to build mount for /etc/hosts: %v", err)
}
if etcHostMount != nil {
// erase the extra_hosts field if we have a mount so we don't get
// conflicting options error from dockerd
driverConfig.ExtraHosts = nil
hostConfig.Mounts = append(hostConfig.Mounts, docker.HostMount{
Target: etcHostMount.TaskPath,
Source: etcHostMount.HostPath,
Type: "bind",
ReadOnly: etcHostMount.Readonly,
BindOptions: &docker.BindOptions{
Propagation: etcHostMount.PropagationMode,
},
})
}
}

// Setup DNS
// If task DNS options are configured Nomad will manage the resolv.conf file
// Docker driver dns options are not compatible with task dns options
Expand Down
76 changes: 76 additions & 0 deletions drivers/shared/hostnames/mount.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package hostnames

import (
"fmt"
"io/ioutil"
"net"
"path/filepath"
"strings"

"github.com/hashicorp/nomad/plugins/drivers"
)

// GenerateEtcHostsMount writes a /etc/hosts file using the network spec's
// hosts configuration, and returns a mount config so that task drivers can
// bind-mount it into the resulting task's filesystem. The extraHosts
// parameter is expected to be the same format as the extra_hosts field from
// the Docker or containerd drivers: []string{"<hostname>:<ip address>"}
func GenerateEtcHostsMount(taskDir string, conf *drivers.NetworkIsolationSpec, extraHosts []string) (*drivers.MountConfig, error) {
if conf == nil || conf.Mode != drivers.NetIsolationModeGroup {
return nil, nil
}
hostsCfg := conf.HostsConfig
if hostsCfg == nil || hostsCfg.Address == "" || hostsCfg.Hostname == "" {
return nil, nil
}

var content strings.Builder
fmt.Fprintf(&content, `# this file was generated by Nomad
127.0.0.1 localhost
::1 localhost
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
ff02::3 ip6-allhosts
# this entry is the IP address and hostname of the allocation
# shared with tasks in the task group's network
%s %s
`, hostsCfg.Address, hostsCfg.Hostname)

if len(extraHosts) > 0 {
content.WriteString("\n# these entries are extra hosts added by the task config")
for _, hostLine := range extraHosts {
hostsEntry := strings.SplitN(hostLine, ":", 2)
if len(hostsEntry) != 2 {
return nil, fmt.Errorf("invalid hosts entry %q", hostLine)
}
if net.ParseIP(hostsEntry[1]) == nil {
return nil, fmt.Errorf("invalid IP address %q", hostLine)
}
content.WriteString(fmt.Sprintf("\n%s %s", hostsEntry[1], hostsEntry[0]))
}
content.WriteString("\n")
}

path := filepath.Join(taskDir, "hosts")
err := ioutil.WriteFile(path, []byte(content.String()), 0644)
if err != nil {
return nil, err
}

// Note that we're not setting readonly. The file is in the task dir
// anyways, so this lets the task overwrite its own hosts file if the
// application knows better than Nomad here. Task drivers may override
// this behavior.
mount := &drivers.MountConfig{
TaskPath: "/etc/hosts",
HostPath: path,
Readonly: false,
PropagationMode: "private",
}

return mount, nil
}
120 changes: 120 additions & 0 deletions drivers/shared/hostnames/mount_unix_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// +build !windows

package hostnames

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/hashicorp/nomad/plugins/drivers"
"github.com/stretchr/testify/require"
)

func TestGenerateEtcHostsMount(t *testing.T) {

testCases := []struct {
name string
spec *drivers.NetworkIsolationSpec
extraHosts []string
expected []string
expectedErr string
}{
{
name: "no-spec",
},
{
name: "no-hosts-config",
spec: &drivers.NetworkIsolationSpec{Mode: drivers.NetIsolationModeGroup},
},
{
name: "base-case",
spec: &drivers.NetworkIsolationSpec{
Mode: drivers.NetIsolationModeGroup,
HostsConfig: &drivers.HostsConfig{
Address: "192.168.1.1",
Hostname: "xyzzy",
},
},
expected: []string{
"192.168.1.1 xyzzy",
},
},
{
name: "with-valid-extra-hosts",
spec: &drivers.NetworkIsolationSpec{
Mode: drivers.NetIsolationModeGroup,
HostsConfig: &drivers.HostsConfig{
Address: "192.168.1.1",
Hostname: "xyzzy",
},
},
extraHosts: []string{
"apple:192.168.1.2",
"banana:2001:0db8:85a3:0000:0000:8a2e:0370:7334",
},
expected: []string{
"192.168.1.1 xyzzy",
"192.168.1.2 apple",
"2001:0db8:85a3:0000:0000:8a2e:0370:7334 banana",
},
},
{
name: "invalid-extra-hosts-syntax",
spec: &drivers.NetworkIsolationSpec{
Mode: drivers.NetIsolationModeGroup,
HostsConfig: &drivers.HostsConfig{
Address: "192.168.1.1",
Hostname: "xyzzy",
},
},
extraHosts: []string{"apple192.168.1.2"},
expectedErr: "invalid hosts entry \"apple192.168.1.2\"",
},
{
name: "invalid-extra-hosts-bad-ip",
spec: &drivers.NetworkIsolationSpec{
Mode: drivers.NetIsolationModeGroup,
HostsConfig: &drivers.HostsConfig{
Address: "192.168.1.1",
Hostname: "xyzzy",
},
},
extraHosts: []string{"apple:192.168.1.256"},
expectedErr: "invalid IP address \"apple:192.168.1.256\"",
},
}
for _, tc := range testCases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
require := require.New(t)

taskDir, err := ioutil.TempDir("",
fmt.Sprintf("generateEtcHosts_Test-%s", tc.name))
defer os.RemoveAll(taskDir)
require.NoError(err)
dest := filepath.Join(taskDir, "hosts")

got, err := GenerateEtcHostsMount(taskDir, tc.spec, tc.extraHosts)

if tc.expectedErr != "" {
require.EqualError(err, tc.expectedErr)
} else {
require.NoError(err)
}
if len(tc.expected) == 0 {
require.Nil(got)
} else {
require.NotNil(got)
require.FileExists(dest)
tmpHosts, err := ioutil.ReadFile(dest)
require.NoError(err)
for _, line := range tc.expected {
require.Contains(string(tmpHosts), line)
}
}
})
}
}
12 changes: 9 additions & 3 deletions plugins/drivers/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,9 +200,15 @@ var (
)

type NetworkIsolationSpec struct {
Mode NetIsolationMode
Path string
Labels map[string]string
Mode NetIsolationMode
Path string
Labels map[string]string
HostsConfig *HostsConfig
}

type HostsConfig struct {
Hostname string
Address string
}

// MountConfigSupport is an enum that defaults to "all" for backwards
Expand Down
Loading