Merge pull request #2709 from hashicorp/f-advertise-docker-ips

Advertise driver-specific addresses

api/jobs_test.go

@@ -356,9 +356,10 @@ func TestJobs_Canonicalize(t *testing.T) {
 								},
 								Services: []*Service{
 									{
-										Name:      "global-redis-check",
-										Tags:      []string{"global", "cache"},
-										PortLabel: "db",
+										Name:        "global-redis-check",
+										Tags:        []string{"global", "cache"},
+										PortLabel:   "db",
+										AddressMode: "auto",
 										Checks: []ServiceCheck{
 											{
 												Name:     "alive",

api/tasks.go

@@ -95,17 +95,23 @@ type ServiceCheck struct {
 
 // The Service model represents a Consul service definition
 type Service struct {
-	Id        string
-	Name      string
-	Tags      []string
-	PortLabel string `mapstructure:"port"`
-	Checks    []ServiceCheck
+	Id          string
+	Name        string
+	Tags        []string
+	PortLabel   string `mapstructure:"port"`
+	AddressMode string `mapstructure:"address_mode"`
+	Checks      []ServiceCheck
 }
 
 func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
 	if s.Name == "" {
 		s.Name = fmt.Sprintf("%s-%s-%s", *job.Name, *tg.Name, t.Name)
 	}
+
+	// Default to AddressModeAuto
+	if s.AddressMode == "" {
+		s.AddressMode = "auto"
+	}
 }
 
 // EphemeralDisk is an ephemeral disk object

client/consul.go

@@ -2,13 +2,14 @@ package client
 
 import (
 	"github.com/hashicorp/nomad/client/driver"
+	cstructs "github.com/hashicorp/nomad/client/structs"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 // ConsulServiceAPI is the interface the Nomad Client uses to register and
 // remove services and checks from Consul.
 type ConsulServiceAPI interface {
-	RegisterTask(allocID string, task *structs.Task, exec driver.ScriptExecutor) error
+	RegisterTask(allocID string, task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error
 	RemoveTask(allocID string, task *structs.Task)
-	UpdateTask(allocID string, existing, newTask *structs.Task, exec driver.ScriptExecutor) error
+	UpdateTask(allocID string, existing, newTask *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error
 }

client/consul_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/nomad/client/driver"
+	cstructs "github.com/hashicorp/nomad/client/structs"
 	"github.com/hashicorp/nomad/nomad/structs"
 )
 
@@ -18,9 +19,10 @@ type mockConsulOp struct {
 	allocID string
 	task    *structs.Task
 	exec    driver.ScriptExecutor
+	net     *cstructs.DriverNetwork
 }
 
-func newMockConsulOp(op, allocID string, task *structs.Task, exec driver.ScriptExecutor) mockConsulOp {
+func newMockConsulOp(op, allocID string, task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) mockConsulOp {
 	if op != "add" && op != "remove" && op != "update" {
 		panic(fmt.Errorf("invalid consul op: %s", op))
 	}
@@ -29,6 +31,7 @@ func newMockConsulOp(op, allocID string, task *structs.Task, exec driver.ScriptE
 		allocID: allocID,
 		task:    task,
 		exec:    exec,
+		net:     net,
 	}
 }
 
@@ -52,25 +55,25 @@ func newMockConsulServiceClient() *mockConsulServiceClient {
 	return &m
 }
 
-func (m *mockConsulServiceClient) UpdateTask(allocID string, old, new *structs.Task, exec driver.ScriptExecutor) error {
+func (m *mockConsulServiceClient) UpdateTask(allocID string, old, new *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	m.logger.Printf("[TEST] mock_consul: UpdateTask(%q, %v, %v, %T)", allocID, old, new, exec)
-	m.ops = append(m.ops, newMockConsulOp("update", allocID, new, exec))
+	m.logger.Printf("[TEST] mock_consul: UpdateTask(%q, %v, %v, %T, %x)", allocID, old, new, exec, net.Hash())
+	m.ops = append(m.ops, newMockConsulOp("update", allocID, new, exec, net))
 	return nil
 }
 
-func (m *mockConsulServiceClient) RegisterTask(allocID string, task *structs.Task, exec driver.ScriptExecutor) error {
+func (m *mockConsulServiceClient) RegisterTask(allocID string, task *structs.Task, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	m.logger.Printf("[TEST] mock_consul: RegisterTask(%q, %q, %T)", allocID, task.Name, exec)
-	m.ops = append(m.ops, newMockConsulOp("add", allocID, task, exec))
+	m.logger.Printf("[TEST] mock_consul: RegisterTask(%q, %q, %T, %x)", allocID, task.Name, exec, net.Hash())
+	m.ops = append(m.ops, newMockConsulOp("add", allocID, task, exec, net))
 	return nil
 }
 
 func (m *mockConsulServiceClient) RemoveTask(allocID string, task *structs.Task) {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	m.logger.Printf("[TEST] mock_consul: RemoveTask(%q, %q)", allocID, task.Name)
-	m.ops = append(m.ops, newMockConsulOp("remove", allocID, task, nil))
+	m.ops = append(m.ops, newMockConsulOp("remove", allocID, task, nil, nil))
 }

client/driver/docker.go

@@ -471,7 +471,7 @@ func (d *DockerDriver) Prestart(ctx *ExecContext, task *structs.Task) (*Prestart
 		return nil, err
 	}
 
-	// Set state needed by Start()
+	// Set state needed by Start
 	d.driverConfig = driverConfig
 
 	// Initialize docker API clients
@@ -485,15 +485,21 @@ func (d *DockerDriver) Prestart(ctx *ExecContext, task *structs.Task) (*Prestart
 	if err != nil {
 		return nil, err
 	}
+	d.imageID = id
 
 	resp := NewPrestartResponse()
 	resp.CreatedResources.Add(dockerImageResKey, id)
-	resp.PortMap = d.driverConfig.PortMap
-	d.imageID = id
+
+	// Return the PortMap if it's set
+	if len(driverConfig.PortMap) > 0 {
+		resp.Network = &cstructs.DriverNetwork{
+			PortMap: driverConfig.PortMap,
+		}
+	}
 	return resp, nil
 }
 
-func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
+func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
 
 	pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
 	executorConfig := &dstructs.ExecutorConfig{
@@ -560,6 +566,15 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle
 			pluginClient.Kill()
 			return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err)
 		}
+		// InspectContainer to get all of the container metadata as
+		// much of the metadata (eg networking) isn't populated until
+		// the container is started
+		if container, err = client.InspectContainer(container.ID); err != nil {
+			err = fmt.Errorf("failed to inspect started container %s: %s", container.ID, err)
+			d.logger.Printf("[ERR] driver.docker: %v", err)
+			pluginClient.Kill()
+			return nil, structs.NewRecoverableError(err, true)
+		}
 		d.logger.Printf("[INFO] driver.docker: started container %s", container.ID)
 	} else {
 		d.logger.Printf("[DEBUG] driver.docker: re-attaching to container %s with status %q",
@@ -585,7 +600,58 @@ func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle
 	}
 	go h.collectStats()
 	go h.run()
-	return h, nil
+
+	// Detect container address
+	ip, autoUse := d.detectIP(container)
+
+	// Create a response with the driver handle and container network metadata
+	resp := &StartResponse{
+		Handle: h,
+		Network: &cstructs.DriverNetwork{
+			PortMap:       d.driverConfig.PortMap,
+			IP:            ip,
+			AutoAdvertise: autoUse,
+		},
+	}
+	return resp, nil
+}
+
+// detectIP of Docker container. Returns the first IP found as well as true if
+// the IP should be advertised (bridge network IPs return false). Returns an
+// empty string and false if no IP could be found.
+func (d *DockerDriver) detectIP(c *docker.Container) (string, bool) {
+	if c.NetworkSettings == nil {
+		// This should only happen if there's been a coding error (such
+		// as not calling InspetContainer after CreateContainer). Code
+		// defensively in case the Docker API changes subtly.
+		d.logger.Printf("[ERROR] driver.docker: no network settings for container %s", c.ID)
+		return "", false
+	}
+
+	ip, ipName := "", ""
+	auto := false
+	for name, net := range c.NetworkSettings.Networks {
+		if net.IPAddress == "" {
+			// Ignore networks without an IP address
+			continue
+		}
+
+		ip = net.IPAddress
+		ipName = name
+
+		// Don't auto-advertise bridge IPs
+		if name != "bridge" {
+			auto = true
+		}
+
+		break
+	}
+
+	if n := len(c.NetworkSettings.Networks); n > 1 {
+		d.logger.Printf("[WARN] driver.docker: multiple (%d) Docker networks for container %q but Nomad only supports 1: choosing %q", n, c.ID, ipName)
+	}
+
+	return ip, auto
 }
 
 func (d *DockerDriver) Cleanup(_ *ExecContext, res *CreatedResources) error {