Merge branch 'master' of github.com:hashicorp/nomad into f-1157-valid…

…ate-node-meta-variables

GNUmakefile

@@ -16,14 +16,9 @@ ifeq (,$(findstring $(THIS_OS),Darwin Linux FreeBSD))
 $(error Building Nomad is currently only supported on Darwin and Linux.)
 endif
 
-# On Linux we build for Linux, Windows, and potentially Linux+LXC
+# On Linux we build for Linux and Windows
 ifeq (Linux,$(THIS_OS))
 
-# Detect if we have LXC on the path
-ifeq (0,$(shell pkg-config --exists lxc; echo $$?))
-HAS_LXC="true"
-endif
-
 ifeq ($(TRAVIS),true)
 $(info Running in Travis, verbose mode is disabled)
 else
@@ -38,9 +33,6 @@ ALL_TARGETS += linux_386 \
 	windows_386 \
 	windows_amd64
 
-ifeq ("true",$(HAS_LXC))
-ALL_TARGETS += linux_amd64-lxc
-endif
 endif
 
 # On MacOS, we only build for MacOS
@@ -122,14 +114,6 @@ pkg/windows_amd64/nomad: $(SOURCE_FILES) ## Build Nomad for windows/amd64
 		-tags "$(GO_TAGS)" \
 		-o "$@.exe"
 
-pkg/linux_amd64-lxc/nomad: $(SOURCE_FILES) ## Build Nomad+LXC for linux/amd64
-	@echo "==> Building $@ with tags $(GO_TAGS)..."
-	@CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \
-		go build \
-		-ldflags $(GO_LDFLAGS) \
-		-tags "$(GO_TAGS) lxc" \
-		-o "$@"
-
 # Define package targets for each of the build targets we actually have on this system
 define makePackageTarget
 
@@ -222,7 +206,7 @@ changelogfmt:
 dev: GOOS=$(shell go env GOOS)
 dev: GOARCH=$(shell go env GOARCH)
 dev: GOPATH=$(shell go env GOPATH)
-dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)$(if $(HAS_LXC),-lxc)/nomad
+dev: DEV_TARGET=pkg/$(GOOS)_$(GOARCH)/nomad
 dev: vendorfmt changelogfmt ## Build for the current development platform
 	@echo "==> Removing old development build..."
 	@rm -f $(PROJECT_ROOT)/$(DEV_TARGET)
@@ -268,7 +252,7 @@ test-nomad: dev ## Run Nomad test suites
 		$(if $(ENABLE_RACE),-race) $(if $(VERBOSE),-v) \
 		-cover \
 		-timeout=15m \
-		-tags="$(if $(HAS_LXC),lxc)" ./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
+		./... $(if $(VERBOSE), >test.log ; echo $$? > exit-code)
 	@if [ $(VERBOSE) ] ; then \
 		bash -C "$(PROJECT_ROOT)/scripts/test_check.sh" ; \
 	fi

api/nodes.go

@@ -124,12 +124,12 @@ func (n *Nodes) MonitorDrain(ctx context.Context, nodeID string, index uint64, i
 	allocCh := make(chan *MonitorMessage, 8)
 
 	// Multiplex node and alloc chans onto outCh. This goroutine closes
-	// outCh when other chans have been closed or context canceled.
+	// outCh when other chans have been closed.
 	multiplexCtx, cancel := context.WithCancel(ctx)
 	go n.monitorDrainMultiplex(multiplexCtx, cancel, outCh, nodeCh, allocCh)
 
 	// Monitor node for updates
-	go n.monitorDrainNode(multiplexCtx, cancel, nodeID, index, nodeCh)
+	go n.monitorDrainNode(multiplexCtx, nodeID, index, nodeCh)
 
 	// Monitor allocs on node for updates
 	go n.monitorDrainAllocs(multiplexCtx, nodeID, ignoreSys, allocCh)
@@ -160,12 +160,14 @@ func (n *Nodes) monitorDrainMultiplex(ctx context.Context, cancel func(),
 			if !nodeOk {
 				// nil chan to prevent further recvs
 				nodeCh = nil
+				continue
 			}
 
 		case msg, allocOk = <-allocCh:
 			if !allocOk {
 				// nil chan to prevent further recvs
 				allocCh = nil
+				continue
 			}
 
 		case <-ctx.Done():
@@ -179,14 +181,6 @@ func (n *Nodes) monitorDrainMultiplex(ctx context.Context, cancel func(),
 		select {
 		case outCh <- msg:
 		case <-ctx.Done():
-
-			// If we are exiting but we have a message, attempt to send it
-			// so we don't lose a message but do not block.
-			select {
-			case outCh <- msg:
-			default:
-			}
-
 			return
 		}
 
@@ -199,12 +193,12 @@ func (n *Nodes) monitorDrainMultiplex(ctx context.Context, cancel func(),
 
 // monitorDrainNode emits node updates on nodeCh and closes the channel when
 // the node has finished draining.
-func (n *Nodes) monitorDrainNode(ctx context.Context, cancel func(),
-	nodeID string, index uint64, nodeCh chan<- *MonitorMessage) {
+func (n *Nodes) monitorDrainNode(ctx context.Context, nodeID string,
+	index uint64, nodeCh chan<- *MonitorMessage) {
+
 	defer close(nodeCh)
 
 	var lastStrategy *DrainStrategy
-	var strategyChanged bool
 	q := QueryOptions{
 		AllowStale: true,
 		WaitIndex:  index,
@@ -222,12 +216,7 @@ func (n *Nodes) monitorDrainNode(ctx context.Context, cancel func(),
 
 		if node.DrainStrategy == nil {
 			var msg *MonitorMessage
-			if strategyChanged {
-				msg = Messagef(MonitorMsgLevelInfo, "Node %q has marked all allocations for migration", nodeID)
-			} else {
-				msg = Messagef(MonitorMsgLevelInfo, "No drain strategy set for node %s", nodeID)
-				defer cancel()
-			}
+			msg = Messagef(MonitorMsgLevelInfo, "Drain complete for node %s", nodeID)
 			select {
 			case nodeCh <- msg:
 			case <-ctx.Done():
@@ -254,7 +243,6 @@ func (n *Nodes) monitorDrainNode(ctx context.Context, cancel func(),
 		}
 
 		lastStrategy = node.DrainStrategy
-		strategyChanged = true
 
 		// Drain still ongoing, update index and block for updates
 		q.WaitIndex = meta.LastIndex

client/allocdir/alloc_dir_test.go

@@ -14,7 +14,6 @@ import (
 	"syscall"
 	"testing"
 
-	cstructs "github.com/hashicorp/nomad/client/structs"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/nomad/structs"
 	"github.com/stretchr/testify/require"
@@ -112,11 +111,11 @@ func TestAllocDir_MountSharedAlloc(t *testing.T) {
 
 	// Build 2 task dirs
 	td1 := d.NewTaskDir(t1.Name)
-	if err := td1.Build(false, nil, cstructs.FSIsolationChroot); err != nil {
+	if err := td1.Build(true, nil); err != nil {
 		t.Fatalf("error build task=%q dir: %v", t1.Name, err)
 	}
 	td2 := d.NewTaskDir(t2.Name)
-	if err := td2.Build(false, nil, cstructs.FSIsolationChroot); err != nil {
+	if err := td2.Build(true, nil); err != nil {
 		t.Fatalf("error build task=%q dir: %v", t2.Name, err)
 	}
 
@@ -157,11 +156,11 @@ func TestAllocDir_Snapshot(t *testing.T) {
 
 	// Build 2 task dirs
 	td1 := d.NewTaskDir(t1.Name)
-	if err := td1.Build(false, nil, cstructs.FSIsolationImage); err != nil {
+	if err := td1.Build(false, nil); err != nil {
 		t.Fatalf("error build task=%q dir: %v", t1.Name, err)
 	}
 	td2 := d.NewTaskDir(t2.Name)
-	if err := td2.Build(false, nil, cstructs.FSIsolationImage); err != nil {
+	if err := td2.Build(false, nil); err != nil {
 		t.Fatalf("error build task=%q dir: %v", t2.Name, err)
 	}
 
@@ -249,7 +248,7 @@ func TestAllocDir_Move(t *testing.T) {
 	defer d2.Destroy()
 
 	td1 := d1.NewTaskDir(t1.Name)
-	if err := td1.Build(false, nil, cstructs.FSIsolationImage); err != nil {
+	if err := td1.Build(false, nil); err != nil {
 		t.Fatalf("TaskDir.Build() faild: %v", err)
 	}
 
@@ -345,7 +344,7 @@ func TestAllocDir_ReadAt_SecretDir(t *testing.T) {
 	defer d.Destroy()
 
 	td := d.NewTaskDir(t1.Name)
-	if err := td.Build(false, nil, cstructs.FSIsolationImage); err != nil {
+	if err := td.Build(false, nil); err != nil {
 		t.Fatalf("TaskDir.Build() failed: %v", err)
 	}
 

client/allocdir/task_dir.go

@@ -7,7 +7,6 @@ import (
 	"path/filepath"
 
 	hclog "github.com/hashicorp/go-hclog"
-	cstructs "github.com/hashicorp/nomad/client/structs"
 )
 
 // TaskDir contains all of the paths relevant to a task. All paths are on the
@@ -75,7 +74,7 @@ func (t *TaskDir) Copy() *TaskDir {
 // Build default directories and permissions in a task directory. chrootCreated
 // allows skipping chroot creation if the caller knows it has already been
 // done.
-func (t *TaskDir) Build(chrootCreated bool, chroot map[string]string, fsi cstructs.FSIsolation) error {
+func (t *TaskDir) Build(createChroot bool, chroot map[string]string) error {
 	if err := os.MkdirAll(t.Dir, 0777); err != nil {
 		return err
 	}
@@ -110,7 +109,7 @@ func (t *TaskDir) Build(chrootCreated bool, chroot map[string]string, fsi cstruc
 	// Image based isolation will bind the shared alloc dir in the driver.
 	// If there's no isolation the task will use the host path to the
 	// shared alloc dir.
-	if fsi == cstructs.FSIsolationChroot {
+	if createChroot {
 		// If the path doesn't exist OR it exists and is empty, link it
 		empty, _ := pathEmpty(t.SharedTaskDir)
 		if !pathExists(t.SharedTaskDir) || empty {
@@ -130,8 +129,8 @@ func (t *TaskDir) Build(chrootCreated bool, chroot map[string]string, fsi cstruc
 	}
 
 	// Build chroot if chroot filesystem isolation is going to be used
-	if fsi == cstructs.FSIsolationChroot {
-		if err := t.buildChroot(chrootCreated, chroot); err != nil {
+	if createChroot {
+		if err := t.buildChroot(chroot); err != nil {
 			return err
 		}
 	}
@@ -142,15 +141,9 @@ func (t *TaskDir) Build(chrootCreated bool, chroot map[string]string, fsi cstruc
 // buildChroot takes a mapping of absolute directory or file paths on the host
 // to their intended, relative location within the task directory. This
 // attempts hardlink and then defaults to copying. If the path exists on the
-// host and can't be embedded an error is returned. If chrootCreated is true
-// skip expensive embedding operations and only ephemeral operations (eg
-// mounting /dev) are done.
-func (t *TaskDir) buildChroot(chrootCreated bool, entries map[string]string) error {
-	if !chrootCreated {
-		// Link/copy chroot entries
-		return t.embedDirs(entries)
-	}
-	return nil
+// host and can't be embedded an error is returned.
+func (t *TaskDir) buildChroot(entries map[string]string) error {
+	return t.embedDirs(entries)
 }
 
 func (t *TaskDir) embedDirs(entries map[string]string) error {

client/allocdir/task_dir_test.go

@@ -6,7 +6,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	cstructs "github.com/hashicorp/nomad/client/structs"
 	"github.com/hashicorp/nomad/helper/testlog"
 )
 
@@ -104,7 +103,7 @@ func TestTaskDir_NonRoot_Image(t *testing.T) {
 		t.Fatalf("Build() failed: %v", err)
 	}
 
-	if err := td.Build(false, nil, cstructs.FSIsolationImage); err != nil {
+	if err := td.Build(false, nil); err != nil {
 		t.Fatalf("TaskDir.Build failed: %v", err)
 	}
 }
@@ -127,7 +126,7 @@ func TestTaskDir_NonRoot(t *testing.T) {
 		t.Fatalf("Build() failed: %v", err)
 	}
 
-	if err := td.Build(false, nil, cstructs.FSIsolationNone); err != nil {
+	if err := td.Build(false, nil); err != nil {
 		t.Fatalf("TaskDir.Build failed: %v", err)
 	}
 

client/allocrunner/alloc_runner.go

@@ -458,7 +458,23 @@ func (ar *allocRunner) handleTaskStateUpdates() {
 				ar.logger.Debug("task failure, destroying all tasks", "failed_task", killTask)
 			}
 
+			// Emit kill event for live runners
+			for _, tr := range liveRunners {
+				tr.EmitEvent(killEvent)
+			}
+
+			// Kill 'em all
 			states = ar.killTasks()
+
+			// Wait for TaskRunners to exit before continuing to
+			// prevent looping before TaskRunners have transitioned
+			// to Dead.
+			for _, tr := range liveRunners {
+				select {
+				case <-tr.WaitCh():
+				case <-ar.waitCh:
+				}
+			}
 		}
 
 		// Get the client allocation
@@ -485,7 +501,7 @@ func (ar *allocRunner) killTasks() map[string]*structs.TaskState {
 			continue
 		}
 
-		err := tr.Kill(context.TODO(), structs.NewTaskEvent(structs.TaskKilled))
+		err := tr.Kill(context.TODO(), structs.NewTaskEvent(structs.TaskKilling))
 		if err != nil && err != taskrunner.ErrTaskNotRunning {
 			ar.logger.Warn("error stopping leader task", "error", err, "task_name", name)
 		}
@@ -505,7 +521,7 @@ func (ar *allocRunner) killTasks() map[string]*structs.TaskState {
 		wg.Add(1)
 		go func(name string, tr *taskrunner.TaskRunner) {
 			defer wg.Done()
-			err := tr.Kill(context.TODO(), structs.NewTaskEvent(structs.TaskKilled))
+			err := tr.Kill(context.TODO(), structs.NewTaskEvent(structs.TaskKilling))
 			if err != nil && err != taskrunner.ErrTaskNotRunning {
 				ar.logger.Warn("error stopping task", "error", err, "task_name", name)
 			}

client/allocrunner/interfaces/task_lifecycle.go

@@ -96,7 +96,7 @@ type TaskPoststartRequest struct {
 	DriverExec interfaces.ScriptExecutor
 
 	// Network info (may be nil)
-	DriverNetwork *cstructs.DriverNetwork
+	DriverNetwork *drivers.DriverNetwork
 
 	// TaskEnv is the task's environment
 	TaskEnv *taskenv.TaskEnv

client/allocrunner/taskrunner/dispatch_hook_test.go

@@ -9,7 +9,6 @@ import (
 	"github.com/golang/snappy"
 	"github.com/hashicorp/nomad/client/allocdir"
 	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
-	cstructs "github.com/hashicorp/nomad/client/structs"
 	"github.com/hashicorp/nomad/helper/testlog"
 	"github.com/hashicorp/nomad/nomad/mock"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -34,7 +33,7 @@ func TestTaskRunner_DispatchHook_NoPayload(t *testing.T) {
 	alloc := mock.BatchAlloc()
 	task := alloc.Job.TaskGroups[0].Tasks[0]
 	taskDir := allocDir.NewTaskDir(task.Name)
-	require.NoError(taskDir.Build(false, nil, cstructs.FSIsolationNone))
+	require.NoError(taskDir.Build(false, nil))
 
 	h := newDispatchHook(alloc, logger)
 
@@ -79,7 +78,7 @@ func TestTaskRunner_DispatchHook_Ok(t *testing.T) {
 		File: "out",
 	}
 	taskDir := allocDir.NewTaskDir(task.Name)
-	require.NoError(taskDir.Build(false, nil, cstructs.FSIsolationNone))
+	require.NoError(taskDir.Build(false, nil))
 
 	h := newDispatchHook(alloc, logger)
 
@@ -123,7 +122,7 @@ func TestTaskRunner_DispatchHook_Error(t *testing.T) {
 		File: "out",
 	}
 	taskDir := allocDir.NewTaskDir(task.Name)
-	require.NoError(taskDir.Build(false, nil, cstructs.FSIsolationNone))
+	require.NoError(taskDir.Build(false, nil))
 
 	h := newDispatchHook(alloc, logger)
 

client/allocrunner/taskrunner/driver_handle.go

@@ -10,7 +10,7 @@ import (
 )
 
 // NewDriverHandle returns a handle for task operations on a specific task
-func NewDriverHandle(driver drivers.DriverPlugin, taskID string, task *structs.Task, net *cstructs.DriverNetwork) *DriverHandle {
+func NewDriverHandle(driver drivers.DriverPlugin, taskID string, task *structs.Task, net *drivers.DriverNetwork) *DriverHandle {
 	return &DriverHandle{
 		driver: driver,
 		net:    net,
@@ -23,7 +23,7 @@ func NewDriverHandle(driver drivers.DriverPlugin, taskID string, task *structs.T
 // an api to perform driver operations on the task
 type DriverHandle struct {
 	driver drivers.DriverPlugin
-	net    *cstructs.DriverNetwork
+	net    *drivers.DriverNetwork
 	task   *structs.Task
 	taskID string
 }
@@ -61,6 +61,6 @@ func (h *DriverHandle) Exec(timeout time.Duration, cmd string, args []string) ([
 	return res.Stdout, res.ExitResult.ExitCode, res.ExitResult.Err
 }
 
-func (h *DriverHandle) Network() *cstructs.DriverNetwork {
+func (h *DriverHandle) Network() *drivers.DriverNetwork {
 	return h.net
 }