Merge network namespaces work into master (#6046)

Merge network namespaces work into master

api/allocations.go

@@ -458,7 +458,8 @@ type AllocatedTaskResources struct {
 }
 
 type AllocatedSharedResources struct {
-	DiskMB int64
+	DiskMB   int64
+	Networks []*NetworkResource
 }
 
 type AllocatedCpuResources struct {

api/compose_test.go

@@ -20,7 +20,7 @@ func TestCompose(t *testing.T) {
 				{
 					CIDR:          "0.0.0.0/0",
 					MBits:         intToPtr(100),
-					ReservedPorts: []Port{{"", 80}, {"", 443}},
+					ReservedPorts: []Port{{"", 80, 0}, {"", 443, 0}},
 				},
 			},
 		})
@@ -111,8 +111,8 @@ func TestCompose(t *testing.T) {
 									CIDR:  "0.0.0.0/0",
 									MBits: intToPtr(100),
 									ReservedPorts: []Port{
-										{"", 80},
-										{"", 443},
+										{"", 80, 0},
+										{"", 443, 0},
 									},
 								},
 							},

api/resources.go

@@ -86,17 +86,20 @@ func (r *Resources) Merge(other *Resources) {
 type Port struct {
 	Label string
 	Value int `mapstructure:"static"`
+	To    int `mapstructure:"to"`
 }
 
 // NetworkResource is used to describe required network
 // resources of a given task.
 type NetworkResource struct {
+	Mode          string
 	Device        string
 	CIDR          string
 	IP            string
 	MBits         *int
 	ReservedPorts []Port
 	DynamicPorts  []Port
+	Services      []*Service
 }
 
 func (n *NetworkResource) Canonicalize() {

api/tasks.go

@@ -372,6 +372,7 @@ type Service struct {
 	AddressMode  string   `mapstructure:"address_mode"`
 	Checks       []ServiceCheck
 	CheckRestart *CheckRestart `mapstructure:"check_restart"`
+	Connect      *ConsulConnect
 }
 
 func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
@@ -392,6 +393,25 @@ func (s *Service) Canonicalize(t *Task, tg *TaskGroup, job *Job) {
 	}
 }
 
+type ConsulConnect struct {
+	SidecarService *ConsulSidecarService `mapstructure:"sidecar_service"`
+}
+
+type ConsulSidecarService struct {
+	Port  string
+	Proxy *ConsulProxy
+}
+
+type ConsulProxy struct {
+	Upstreams []*ConsulUpstream
+}
+
+type ConsulUpstream struct {
+	//FIXME Pointers?
+	DestinationName string `mapstructure:"destination_name"`
+	LocalBindPort   int    `mapstructure:"local_bind_port"`
+}
+
 // EphemeralDisk is an ephemeral disk object
 type EphemeralDisk struct {
 	Sticky  *bool
@@ -493,7 +513,9 @@ type TaskGroup struct {
 	EphemeralDisk    *EphemeralDisk
 	Update           *UpdateStrategy
 	Migrate          *MigrateStrategy
+	Networks         []*NetworkResource
 	Meta             map[string]string
+	Services         []*Service
 }
 
 // NewTaskGroup creates a new TaskGroup.
@@ -604,6 +626,9 @@ func (g *TaskGroup) Canonicalize(job *Job) {
 	for _, a := range g.Affinities {
 		a.Canonicalize()
 	}
+	for _, n := range g.Networks {
+		n.Canonicalize()
+	}
 }
 
 // Constrain is used to add a constraint to a task group.

api/tasks_test.go

@@ -269,7 +269,7 @@ func TestTask_Require(t *testing.T) {
 			{
 				CIDR:          "0.0.0.0/0",
 				MBits:         intToPtr(100),
-				ReservedPorts: []Port{{"", 80}, {"", 443}},
+				ReservedPorts: []Port{{"", 80, 0}, {"", 443, 0}},
 			},
 		},
 	}

client/allochealth/tracker.go

@@ -238,7 +238,12 @@ func (t *Tracker) watchTaskEvents() {
 		// Store the task states
 		t.l.Lock()
 		for task, state := range alloc.TaskStates {
-			t.taskHealth[task].state = state
+			//TODO(schmichael) for now skip unknown tasks as
+			//they're task group services which don't currently
+			//support checks anyway
+			if v, ok := t.taskHealth[task]; ok {
+				v.state = state
+			}
 		}
 		t.l.Unlock()
 
@@ -355,7 +360,12 @@ OUTER:
 		// Store the task registrations
 		t.l.Lock()
 		for task, reg := range allocReg.Tasks {
-			t.taskHealth[task].taskRegistrations = reg
+			//TODO(schmichael) for now skip unknown tasks as
+			//they're task group services which don't currently
+			//support checks anyway
+			if v, ok := t.taskHealth[task]; ok {
+				v.taskRegistrations = reg
+			}
 		}
 		t.l.Unlock()
 

client/allocrunner/alloc_runner.go

@@ -185,7 +185,9 @@ func NewAllocRunner(config *Config) (*allocRunner, error) {
 	ar.allocDir = allocdir.NewAllocDir(ar.logger, filepath.Join(config.ClientConfig.AllocDir, alloc.ID))
 
 	// Initialize the runners hooks.
-	ar.initRunnerHooks()
+	if err := ar.initRunnerHooks(config.ClientConfig); err != nil {
+		return nil, err
+	}
 
 	// Create the TaskRunners
 	if err := ar.initTaskRunners(tg.Tasks); err != nil {

client/allocrunner/alloc_runner_hooks.go

@@ -6,9 +6,28 @@ import (
 
 	multierror "github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	clientconfig "github.com/hashicorp/nomad/client/config"
 	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/plugins/drivers"
 )
 
+type networkIsolationSetter interface {
+	SetNetworkIsolation(*drivers.NetworkIsolationSpec)
+}
+
+// allocNetworkIsolationSetter is a shim to allow the alloc network hook to
+// set the alloc network isolation configuration without full access
+// to the alloc runner
+type allocNetworkIsolationSetter struct {
+	ar *allocRunner
+}
+
+func (a *allocNetworkIsolationSetter) SetNetworkIsolation(n *drivers.NetworkIsolationSpec) {
+	for _, tr := range a.ar.tasks {
+		tr.SetNetworkIsolation(n)
+	}
+}
+
 // allocHealthSetter is a shim to allow the alloc health watcher hook to set
 // and clear the alloc health without full access to the alloc runner state
 type allocHealthSetter struct {
@@ -76,20 +95,35 @@ func (a *allocHealthSetter) SetHealth(healthy, isDeploy bool, trackerTaskEvents
 }
 
 // initRunnerHooks intializes the runners hooks.
-func (ar *allocRunner) initRunnerHooks() {
+func (ar *allocRunner) initRunnerHooks(config *clientconfig.Config) error {
 	hookLogger := ar.logger.Named("runner_hook")
 
 	// create health setting shim
 	hs := &allocHealthSetter{ar}
 
+	// create network isolation setting shim
+	ns := &allocNetworkIsolationSetter{ar: ar}
+
+	// build the network manager
+	nm, err := newNetworkManager(ar.Alloc(), ar.driverManager)
+	if err != nil {
+		return fmt.Errorf("failed to configure network manager: %v", err)
+	}
+
+	// create network configurator
+	nc := newNetworkConfigurator(ar.Alloc(), config)
+
 	// Create the alloc directory hook. This is run first to ensure the
 	// directory path exists for other hooks.
 	ar.runnerHooks = []interfaces.RunnerHook{
 		newAllocDirHook(hookLogger, ar.allocDir),
 		newUpstreamAllocsHook(hookLogger, ar.prevAllocWatcher),
 		newDiskMigrationHook(hookLogger, ar.prevAllocMigrator, ar.allocDir),
 		newAllocHealthWatcherHook(hookLogger, ar.Alloc(), hs, ar.Listener(), ar.consulClient),
+		newNetworkHook(hookLogger, ns, ar.Alloc(), nm, nc),
 	}
+
+	return nil
 }
 
 // prerun is used to run the runners prerun hooks.

client/allocrunner/network_hook.go

@@ -0,0 +1,88 @@
+package allocrunner
+
+import (
+	"fmt"
+
+	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/plugins/drivers"
+)
+
+// networkHook is an alloc lifecycle hook that manages the network namespace
+// for an alloc
+type networkHook struct {
+	// setter is a callback to set the network isolation spec when after the
+	// network is created
+	setter networkIsolationSetter
+
+	// manager is used when creating the network namespace. This defaults to
+	// bind mounting a network namespace descritor under /var/run/netns but
+	// can be created by a driver if nessicary
+	manager drivers.DriverNetworkManager
+
+	// alloc should only be read from
+	alloc *structs.Allocation
+
+	// spec described the network namespace and is syncronized by specLock
+	spec *drivers.NetworkIsolationSpec
+
+	// networkConfigurator configures the network interfaces, routes, etc once
+	// the alloc network has been created
+	networkConfigurator NetworkConfigurator
+
+	logger hclog.Logger
+}
+
+func newNetworkHook(logger hclog.Logger, ns networkIsolationSetter,
+	alloc *structs.Allocation, netManager drivers.DriverNetworkManager,
+	netConfigurator NetworkConfigurator) *networkHook {
+	return &networkHook{
+		setter:              ns,
+		alloc:               alloc,
+		manager:             netManager,
+		networkConfigurator: netConfigurator,
+		logger:              logger,
+	}
+}
+
+func (h *networkHook) Name() string {
+	return "network"
+}
+
+func (h *networkHook) Prerun() error {
+	tg := h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup)
+	if len(tg.Networks) == 0 || tg.Networks[0].Mode == "host" || tg.Networks[0].Mode == "" {
+		return nil
+	}
+
+	if h.manager == nil || h.networkConfigurator == nil {
+		h.logger.Trace("shared network namespaces are not supported on this platform, skipping network hook")
+		return nil
+	}
+
+	spec, err := h.manager.CreateNetwork(h.alloc.ID)
+	if err != nil {
+		return fmt.Errorf("failed to create network for alloc: %v", err)
+	}
+
+	if spec != nil {
+		h.spec = spec
+		h.setter.SetNetworkIsolation(spec)
+	}
+
+	if err := h.networkConfigurator.Setup(h.alloc, spec); err != nil {
+		return fmt.Errorf("failed to configure networking for alloc: %v", err)
+	}
+	return nil
+}
+
+func (h *networkHook) Postrun() error {
+	if h.spec == nil {
+		return nil
+	}
+
+	if err := h.networkConfigurator.Teardown(h.alloc, h.spec); err != nil {
+		h.logger.Error("failed to cleanup network for allocation, resources may have leaked", "alloc", h.alloc.ID, "error", err)
+	}
+	return h.manager.DestroyNetwork(h.alloc.ID, h.spec)
+}

client/allocrunner/network_hook_test.go

@@ -0,0 +1,86 @@
+package allocrunner
+
+import (
+	"testing"
+
+	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
+	"github.com/hashicorp/nomad/helper/testlog"
+	"github.com/hashicorp/nomad/nomad/mock"
+	"github.com/hashicorp/nomad/nomad/structs"
+	"github.com/hashicorp/nomad/plugins/drivers"
+	"github.com/hashicorp/nomad/plugins/drivers/testutils"
+	"github.com/stretchr/testify/require"
+)
+
+// statically assert network hook implements the expected interfaces
+var _ interfaces.RunnerPrerunHook = (*networkHook)(nil)
+var _ interfaces.RunnerPostrunHook = (*networkHook)(nil)
+
+type mockNetworkIsolationSetter struct {
+	t            *testing.T
+	expectedSpec *drivers.NetworkIsolationSpec
+	called       bool
+}
+
+func (m *mockNetworkIsolationSetter) SetNetworkIsolation(spec *drivers.NetworkIsolationSpec) {
+	m.called = true
+	require.Exactly(m.t, m.expectedSpec, spec)
+}
+
+// Test that the prerun and postrun hooks call the setter with the expected spec when
+// the network mode is not host
+func TestNetworkHook_Prerun_Postrun(t *testing.T) {
+	alloc := mock.Alloc()
+	alloc.Job.TaskGroups[0].Networks = []*structs.NetworkResource{
+		{
+			Mode: "bridge",
+		},
+	}
+	spec := &drivers.NetworkIsolationSpec{
+		Mode:   drivers.NetIsolationModeGroup,
+		Path:   "test",
+		Labels: map[string]string{"abc": "123"},
+	}
+
+	destroyCalled := false
+	nm := &testutils.MockDriver{
+		MockNetworkManager: testutils.MockNetworkManager{
+			CreateNetworkF: func(allocID string) (*drivers.NetworkIsolationSpec, error) {
+				require.Equal(t, alloc.ID, allocID)
+				return spec, nil
+			},
+
+			DestroyNetworkF: func(allocID string, netSpec *drivers.NetworkIsolationSpec) error {
+				destroyCalled = true
+				require.Equal(t, alloc.ID, allocID)
+				require.Exactly(t, spec, netSpec)
+				return nil
+			},
+		},
+	}
+	setter := &mockNetworkIsolationSetter{
+		t:            t,
+		expectedSpec: spec,
+	}
+	require := require.New(t)
+
+	logger := testlog.HCLogger(t)
+	hook := newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{})
+	require.NoError(hook.Prerun())
+	require.True(setter.called)
+	require.False(destroyCalled)
+	require.NoError(hook.Postrun())
+	require.True(destroyCalled)
+
+	// reset and use host network mode
+	setter.called = false
+	destroyCalled = false
+	alloc.Job.TaskGroups[0].Networks[0].Mode = "host"
+	hook = newNetworkHook(logger, setter, alloc, nm, &hostNetworkConfigurator{})
+	require.NoError(hook.Prerun())
+	require.False(setter.called)
+	require.False(destroyCalled)
+	require.NoError(hook.Postrun())
+	require.False(destroyCalled)
+
+}