Service registration for IPv6 docker addresses (Fixes #3785)

CHANGELOG.md

@@ -10,6 +10,7 @@ IMPROVEMENTS:
  * client: Allow '.' in environment variable names [[GH-3760](https://github.com/hashicorp/nomad/issues/3760)]
  * discovery: Allow `check_restart` to be specified in the `service` stanza.
    [[GH-3718](https://github.com/hashicorp/nomad/issues/3718)]
+ * driver/docker: Support advertising IPv6 addresses [[GH-3790](https://github.com/hashicorp/nomad/issues/3790)]
  * driver/docker; Support overriding image entrypoint [[GH-3788](https://github.com/hashicorp/nomad/issues/3788)]
  * driver/docker: Support adding or dropping capabilities [[GH-3754](https://github.com/hashicorp/nomad/issues/3754)]
  * driver/docker: Support mounting root filesystem as read-only [[GH-3802](https://github.com/hashicorp/nomad/issues/3802)]

client/driver/docker.go

@@ -173,50 +173,51 @@ type DockerVolumeDriverConfig struct {
 
 // DockerDriverConfig defines the user specified config block in a jobspec
 type DockerDriverConfig struct {
-	ImageName        string              `mapstructure:"image"`              // Container's Image Name
-	LoadImage        string              `mapstructure:"load"`               // LoadImage is a path to an image archive file
-	Command          string              `mapstructure:"command"`            // The Command to run when the container starts up
-	Args             []string            `mapstructure:"args"`               // The arguments to the Command
-	Entrypoint       []string            `mapstructure:"entrypoint"`         // Override the containers entrypoint
-	IpcMode          string              `mapstructure:"ipc_mode"`           // The IPC mode of the container - host and none
-	NetworkMode      string              `mapstructure:"network_mode"`       // The network mode of the container - host, nat and none
-	NetworkAliases   []string            `mapstructure:"network_aliases"`    // The network-scoped alias for the container
-	IPv4Address      string              `mapstructure:"ipv4_address"`       // The container ipv4 address
-	IPv6Address      string              `mapstructure:"ipv6_address"`       // the container ipv6 address
-	PidMode          string              `mapstructure:"pid_mode"`           // The PID mode of the container - host and none
-	UTSMode          string              `mapstructure:"uts_mode"`           // The UTS mode of the container - host and none
-	UsernsMode       string              `mapstructure:"userns_mode"`        // The User namespace mode of the container - host and none
-	PortMapRaw       []map[string]string `mapstructure:"port_map"`           //
-	PortMap          map[string]int      `mapstructure:"-"`                  // A map of host port labels and the ports exposed on the container
-	Privileged       bool                `mapstructure:"privileged"`         // Flag to run the container in privileged mode
-	SysctlRaw        []map[string]string `mapstructure:"sysctl"`             //
-	Sysctl           map[string]string   `mapstructure:"-"`                  // The sysctl custom configurations
-	UlimitRaw        []map[string]string `mapstructure:"ulimit"`             //
-	Ulimit           []docker.ULimit     `mapstructure:"-"`                  // The ulimit custom configurations
-	DNSServers       []string            `mapstructure:"dns_servers"`        // DNS Server for containers
-	DNSSearchDomains []string            `mapstructure:"dns_search_domains"` // DNS Search domains for containers
-	DNSOptions       []string            `mapstructure:"dns_options"`        // DNS Options
-	ExtraHosts       []string            `mapstructure:"extra_hosts"`        // Add host to /etc/hosts (host:IP)
-	Hostname         string              `mapstructure:"hostname"`           // Hostname for containers
-	LabelsRaw        []map[string]string `mapstructure:"labels"`             //
-	Labels           map[string]string   `mapstructure:"-"`                  // Labels to set when the container starts up
-	Auth             []DockerDriverAuth  `mapstructure:"auth"`               // Authentication credentials for a private Docker registry
-	AuthSoftFail     bool                `mapstructure:"auth_soft_fail"`     // Soft-fail if auth creds are provided but fail
-	TTY              bool                `mapstructure:"tty"`                // Allocate a Pseudo-TTY
-	Interactive      bool                `mapstructure:"interactive"`        // Keep STDIN open even if not attached
-	ShmSize          int64               `mapstructure:"shm_size"`           // Size of /dev/shm of the container in bytes
-	WorkDir          string              `mapstructure:"work_dir"`           // Working directory inside the container
-	Logging          []DockerLoggingOpts `mapstructure:"logging"`            // Logging options for syslog server
-	Volumes          []string            `mapstructure:"volumes"`            // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container
-	Mounts           []DockerMount       `mapstructure:"mounts"`             // Docker volumes to mount
-	VolumeDriver     string              `mapstructure:"volume_driver"`      // Docker volume driver used for the container's volumes
-	ForcePull        bool                `mapstructure:"force_pull"`         // Always force pull before running image, useful if your tags are mutable
-	MacAddress       string              `mapstructure:"mac_address"`        // Pin mac address to container
-	SecurityOpt      []string            `mapstructure:"security_opt"`       // Flags to pass directly to security-opt
-	Devices          []DockerDevice      `mapstructure:"devices"`            // To allow mounting USB or other serial control devices
-	CapAdd           []string            `mapstructure:"cap_add"`            // Flags to pass directly to cap-add
-	CapDrop          []string            `mapstructure:"cap_drop"`           // Flags to pass directly to cap-drop
-	ReadonlyRootfs   bool                `mapstructure:"readonly_rootfs"`    // Mount the container’s root filesystem as read only
+	ImageName            string              `mapstructure:"image"`                  // Container's Image Name
+	LoadImage            string              `mapstructure:"load"`                   // LoadImage is a path to an image archive file
+	Command              string              `mapstructure:"command"`                // The Command to run when the container starts up
+	Args                 []string            `mapstructure:"args"`                   // The arguments to the Command
+	Entrypoint           []string            `mapstructure:"entrypoint"`             // Override the containers entrypoint
+	IpcMode              string              `mapstructure:"ipc_mode"`               // The IPC mode of the container - host and none
+	NetworkMode          string              `mapstructure:"network_mode"`           // The network mode of the container - host, nat and none
+	NetworkAliases       []string            `mapstructure:"network_aliases"`        // The network-scoped alias for the container
+	IPv4Address          string              `mapstructure:"ipv4_address"`           // The container ipv4 address
+	IPv6Address          string              `mapstructure:"ipv6_address"`           // the container ipv6 address
+	PidMode              string              `mapstructure:"pid_mode"`               // The PID mode of the container - host and none
+	UTSMode              string              `mapstructure:"uts_mode"`               // The UTS mode of the container - host and none
+	UsernsMode           string              `mapstructure:"userns_mode"`            // The User namespace mode of the container - host and none
+	PortMapRaw           []map[string]string `mapstructure:"port_map"`               //
+	PortMap              map[string]int      `mapstructure:"-"`                      // A map of host port labels and the ports exposed on the container
+	Privileged           bool                `mapstructure:"privileged"`             // Flag to run the container in privileged mode
+	SysctlRaw            []map[string]string `mapstructure:"sysctl"`                 //
+	Sysctl               map[string]string   `mapstructure:"-"`                      // The sysctl custom configurations
+	UlimitRaw            []map[string]string `mapstructure:"ulimit"`                 //
+	Ulimit               []docker.ULimit     `mapstructure:"-"`                      // The ulimit custom configurations
+	DNSServers           []string            `mapstructure:"dns_servers"`            // DNS Server for containers
+	DNSSearchDomains     []string            `mapstructure:"dns_search_domains"`     // DNS Search domains for containers
+	DNSOptions           []string            `mapstructure:"dns_options"`            // DNS Options
+	ExtraHosts           []string            `mapstructure:"extra_hosts"`            // Add host to /etc/hosts (host:IP)
+	Hostname             string              `mapstructure:"hostname"`               // Hostname for containers
+	LabelsRaw            []map[string]string `mapstructure:"labels"`                 //
+	Labels               map[string]string   `mapstructure:"-"`                      // Labels to set when the container starts up
+	Auth                 []DockerDriverAuth  `mapstructure:"auth"`                   // Authentication credentials for a private Docker registry
+	AuthSoftFail         bool                `mapstructure:"auth_soft_fail"`         // Soft-fail if auth creds are provided but fail
+	TTY                  bool                `mapstructure:"tty"`                    // Allocate a Pseudo-TTY
+	Interactive          bool                `mapstructure:"interactive"`            // Keep STDIN open even if not attached
+	ShmSize              int64               `mapstructure:"shm_size"`               // Size of /dev/shm of the container in bytes
+	WorkDir              string              `mapstructure:"work_dir"`               // Working directory inside the container
+	Logging              []DockerLoggingOpts `mapstructure:"logging"`                // Logging options for syslog server
+	Volumes              []string            `mapstructure:"volumes"`                // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container
+	Mounts               []DockerMount       `mapstructure:"mounts"`                 // Docker volumes to mount
+	VolumeDriver         string              `mapstructure:"volume_driver"`          // Docker volume driver used for the container's volumes
+	ForcePull            bool                `mapstructure:"force_pull"`             // Always force pull before running image, useful if your tags are mutable
+	MacAddress           string              `mapstructure:"mac_address"`            // Pin mac address to container
+	SecurityOpt          []string            `mapstructure:"security_opt"`           // Flags to pass directly to security-opt
+	Devices              []DockerDevice      `mapstructure:"devices"`                // To allow mounting USB or other serial control devices
+	CapAdd               []string            `mapstructure:"cap_add"`                // Flags to pass directly to cap-add
+	CapDrop              []string            `mapstructure:"cap_drop"`               // Flags to pass directly to cap-drop
+	ReadonlyRootfs       bool                `mapstructure:"readonly_rootfs"`        // Mount the container’s root filesystem as read only
+	AdvertiseIPv6Address bool                `mapstructure:"advertise_ipv6_address"` // Flag to use the GlobalIPv6Address from the container as the detected IP
 }
 
 func sliceMergeUlimit(ulimitsRaw map[string]string) ([]docker.ULimit, error) {
@@ -674,6 +675,9 @@ func (d *DockerDriver) Validate(config map[string]interface{}) error {
 			"readonly_rootfs": {
 				Type: fields.TypeBool,
 			},
+			"advertise_ipv6_address": {
+				Type: fields.TypeBool,
+			},
 		},
 	}
 
@@ -884,6 +888,10 @@ func (d *DockerDriver) detectIP(c *docker.Container) (string, bool) {
 		}
 
 		ip = net.IPAddress
+		if d.driverConfig.AdvertiseIPv6Address {
+			ip = net.GlobalIPv6Address
+			auto = true
+		}
 		ipName = name
 
 		// Don't auto-advertise IPs for default networks (bridge on

client/driver/docker_test.go

@@ -2269,3 +2269,85 @@ func TestDockerDriver_ReadonlyRootfs(t *testing.T) {
 
 	assert.True(t, container.HostConfig.ReadonlyRootfs, "ReadonlyRootfs option not set")
 }
+
+func TestDockerDriver_AdvertiseIPv6Address(t *testing.T) {
+	if !tu.IsTravis() {
+		t.Parallel()
+	}
+	if !testutil.DockerIsConnected(t) {
+		t.Skip("Docker not connected")
+	}
+
+	expectedPrefix := "2001:db8:1::242:ac11"
+	expectedAdvertise := true
+	task := &structs.Task{
+		Name:   "nc-demo",
+		Driver: "docker",
+		Config: map[string]interface{}{
+			"image":   "busybox",
+			"load":    "busybox.tar",
+			"command": "/bin/nc",
+			"args":    []string{"-l", "127.0.0.1", "-p", "0"},
+			"advertise_ipv6_address": expectedAdvertise,
+		},
+		Resources: &structs.Resources{
+			MemoryMB: 256,
+			CPU:      512,
+		},
+		LogConfig: &structs.LogConfig{
+			MaxFiles:      10,
+			MaxFileSizeMB: 10,
+		},
+	}
+
+	client := newTestDockerClient(t)
+
+	// Make sure IPv6 is enabled
+	net, err := client.NetworkInfo("bridge")
+	if err != nil {
+		t.Skip("error retrieving bridge network information, skipping")
+	}
+	if net == nil || !net.EnableIPv6 {
+		t.Skip("IPv6 not enabled on bridge network, skipping")
+	}
+
+	tctx := testDockerDriverContexts(t, task)
+	driver := NewDockerDriver(tctx.DriverCtx)
+	copyImage(t, tctx.ExecCtx.TaskDir, "busybox.tar")
+	defer tctx.AllocDir.Destroy()
+
+	presp, err := driver.Prestart(tctx.ExecCtx, task)
+	defer driver.Cleanup(tctx.ExecCtx, presp.CreatedResources)
+	if err != nil {
+		t.Fatalf("Error in prestart: %v", err)
+	}
+
+	sresp, err := driver.Start(tctx.ExecCtx, task)
+	if err != nil {
+		t.Fatalf("Error in start: %v", err)
+	}
+
+	if sresp.Handle == nil {
+		t.Fatalf("handle is nil\nStack\n%s", debug.Stack())
+	}
+
+	assert.Equal(t, expectedAdvertise, sresp.Network.AutoAdvertise, "Wrong autoadvertise. Expect: %s, got: %s", expectedAdvertise, sresp.Network.AutoAdvertise)
+
+	if !strings.HasPrefix(sresp.Network.IP, expectedPrefix) {
+		t.Fatalf("Got IP address %q want ip address with prefix %q", sresp.Network.IP, expectedPrefix)
+	}
+
+	defer sresp.Handle.Kill()
+	handle := sresp.Handle.(*DockerHandle)
+
+	waitForExist(t, client, handle)
+
+	container, err := client.InspectContainer(handle.ContainerID())
+	if err != nil {
+		t.Fatalf("Error inspecting container: %v", err)
+	}
+
+	if !strings.HasPrefix(container.NetworkSettings.GlobalIPv6Address, expectedPrefix) {
+		t.Fatalf("Got GlobalIPv6address %s want GlobalIPv6address with prefix %s", expectedPrefix, container.NetworkSettings.GlobalIPv6Address)
+	}
+}

scripts/travis-linux.sh

@@ -2,6 +2,10 @@
 
 set -o errexit
 
+#enable ipv6
+echo '{"ipv6":true, "fixed-cidr-v6":"2001:db8:1::/64"}' | sudo tee /etc/docker/daemon.json
+sudo service docker restart
+
 apt-get update
 apt-get install -y liblxc1 lxc-dev lxc shellcheck
 apt-get install -y qemu

website/source/docs/drivers/docker.html.md

@@ -355,6 +355,9 @@ The `docker` driver supports the following configuration in the job spec.  Only
       ]
     }
     ```
+* `advertise_ipv6_address` - (Optional) `true` or `false` (default). Use the container's 
+   IPv6 address (GlobalIPv6Address in Docker) when registering services and checks.
+   See [IPv6 Docker containers](/docs/job-specification/service.html#IPv6 Docker containers) for details.
 
 * `readonly_rootfs` - (Optional) `true` or `false` (default). Mount
   the container's filesystem as read only.

website/source/docs/job-specification/service.html.md

@@ -463,6 +463,109 @@ In this case Nomad doesn't need to assign Redis any host ports. The `service`
 and `check` stanzas can both specify the port number to advertise and check
 directly since Nomad isn't managing any port assignments.
 
+### IPv6 Docker containers
+
+The [Docker](/docs/drivers/docker.html#advertise_ipv6_address) driver supports the
+`advertise_ipv6_address` parameter in it's configuration.
+
+Services will automatically advertise the IPv6 address when `advertise_ipv6_address` 
+is used.
+
+Unlike services, checks do not have an `auto` address mode as there's no way
+for Nomad to know which is the best address to use for checks. Consul needs
+access to the address for any HTTP or TCP checks.
+
+So you have to set `address_mode` parameter in the `check` stanza to `driver`. 
+
+For example using `auto` address mode:
+
+```hcl
+job "example" {
+  datacenters = ["dc1"]
+  group "cache" {
+
+    task "redis" {
+      driver = "docker"
+
+      config {
+        image = "redis:3.2"
+        advertise_ipv6_address = true
+        port_map {
+          db = 6379
+        }
+      }
+
+      resources {
+        cpu    = 500 # 500 MHz
+        memory = 256 # 256MB
+        network {
+          mbits = 10
+          port "db" {}
+        }
+      }
+
+      service {
+        name = "ipv6-redis"
+        port = db
+        check {
+          name     = "ipv6-redis-check"
+          type     = "tcp"
+          interval = "10s"
+          timeout  = "2s"
+          port     = db
+          address_mode = "driver"
+        }
+      }
+    }
+  }
+}
+```
+
+Or using `address_mode=driver` for `service` and `check` with numeric ports:
+
+```hcl
+job "example" {
+  datacenters = ["dc1"]
+  group "cache" {
+
+    task "redis" {
+      driver = "docker"
+
+      config {
+        image = "redis:3.2"
+        advertise_ipv6_address = true
+        # No port map required!
+      }
+
+      resources {
+        cpu    = 500 # 500 MHz
+        memory = 256 # 256MB
+        network {
+          mbits = 10
+        }
+      }
+
+      service {
+        name = "ipv6-redis"
+        port = 6379
+        address_mode = "driver"
+        check {
+          name     = "ipv6-redis-check"
+          type     = "tcp"
+          interval = "10s"
+          timeout  = "2s"
+          port     = 6379
+          address_mode = "driver"
+        }
+      }
+    }
+  }
+}
+```
+
+The `service` and `check` stanzas can both specify the port number to 
+advertise and check directly since Nomad isn't managing any port assignments.
+
 
 - - -