Preemption for system jobs

api/jobs.go

@@ -1024,7 +1024,7 @@ type DesiredUpdates struct {
 	InPlaceUpdate     uint64
 	DestructiveUpdate uint64
 	Canary            uint64
-	Evict             uint64
+	Preemptions       uint64
 }
 
 type JobDispatchRequest struct {

nomad/leader.go

@@ -42,8 +42,12 @@ const (
 
 var minAutopilotVersion = version.Must(version.NewVersion("0.8.0"))
 
-// Default configuration for scheduler with preemption emabled for system jobs
-var defaultSchedulerConfig = &structs.SchedulerConfiguration{PreemptionConfig: structs.PreemptionConfig{SystemSchedulerEnabled: true}}
+// Default configuration for scheduler with preemption enabled for system jobs
+var defaultSchedulerConfig = &structs.SchedulerConfiguration{
+	PreemptionConfig: structs.PreemptionConfig{
+		SystemSchedulerEnabled: true,
+	},
+}
 
 // monitorLeadership is used to monitor if we acquire or lose our role
 // as the leader in the Raft cluster. There is some work the leader is
@@ -1237,7 +1241,8 @@ func (s *Server) getOrCreateAutopilotConfig() *structs.AutopilotConfig {
 	return config
 }
 
-// getOrCreateSchedulerConfig is used to get the scheduler config, initializing it if necessary
+// getOrCreateSchedulerConfig is used to get the scheduler config. We create a default
+// config if it doesn't already exist for bootstrapping an empty cluster
 func (s *Server) getOrCreateSchedulerConfig() *structs.SchedulerConfiguration {
 	state := s.fsm.State()
 	_, config, err := state.SchedulerConfig()

nomad/plan_apply.go

@@ -187,7 +187,7 @@ func (p *planner) applyPlan(plan *structs.Plan, result *structs.PlanResult, snap
 		alloc.ModifyTime = now
 	}
 
-	// Set create and modify time for preempted allocs if any
+	// Set modify time for preempted allocs if any
 	// Also gather jobids to create follow up evals
 	preemptedJobIDs := make(map[structs.NamespacedID]struct{})
 	for _, alloc := range req.NodePreemptions {
@@ -358,9 +358,10 @@ func evaluatePlanPlacements(pool *EvaluatePool, snap *state.StateSnapshot, plan
 		}
 
 		if nodePreemptions := plan.NodePreemptions[nodeID]; nodePreemptions != nil {
-			var filteredNodePreemptions []*structs.Allocation
+
 			// Do a pass over preempted allocs in the plan to check
 			// whether the alloc is already in a terminal state
+			var filteredNodePreemptions []*structs.Allocation
 			for _, preemptedAlloc := range nodePreemptions {
 				alloc, err := snap.AllocByID(nil, preemptedAlloc.ID)
 				if err != nil {

nomad/plan_apply_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/hashicorp/nomad/testutil"
 	"github.com/hashicorp/raft"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -271,6 +272,120 @@ func TestPlanApply_EvalPlan_Simple(t *testing.T) {
 	}
 }
 
+func TestPlanApply_EvalPlan_Preemption(t *testing.T) {
+	t.Parallel()
+	state := testStateStore(t)
+	node := mock.Node()
+	node.NodeResources = &structs.NodeResources{
+		Cpu: structs.NodeCpuResources{
+			CpuShares: 2000,
+		},
+		Memory: structs.NodeMemoryResources{
+			MemoryMB: 4192,
+		},
+		Disk: structs.NodeDiskResources{
+			DiskMB: 30 * 1024,
+		},
+		Networks: []*structs.NetworkResource{
+			{
+				Device: "eth0",
+				CIDR:   "192.168.0.100/32",
+				MBits:  1000,
+			},
+		},
+	}
+	state.UpsertNode(1000, node)
+
+	preemptedAlloc := mock.Alloc()
+	preemptedAlloc.NodeID = node.ID
+	preemptedAlloc.AllocatedResources = &structs.AllocatedResources{
+		Shared: structs.AllocatedSharedResources{
+			DiskMB: 25 * 1024,
+		},
+		Tasks: map[string]*structs.AllocatedTaskResources{
+			"web": {
+				Cpu: structs.AllocatedCpuResources{
+					CpuShares: 1500,
+				},
+				Memory: structs.AllocatedMemoryResources{
+					MemoryMB: 4000,
+				},
+				Networks: []*structs.NetworkResource{
+					{
+						Device:        "eth0",
+						IP:            "192.168.0.100",
+						ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}},
+						MBits:         800,
+						DynamicPorts:  []structs.Port{{Label: "http", Value: 9876}},
+					},
+				},
+			},
+		},
+	}
+
+	// Insert a preempted alloc such that the alloc will fit only after preemption
+	state.UpsertAllocs(1001, []*structs.Allocation{preemptedAlloc})
+
+	alloc := mock.Alloc()
+	alloc.AllocatedResources = &structs.AllocatedResources{
+		Shared: structs.AllocatedSharedResources{
+			DiskMB: 24 * 1024,
+		},
+		Tasks: map[string]*structs.AllocatedTaskResources{
+			"web": {
+				Cpu: structs.AllocatedCpuResources{
+					CpuShares: 1500,
+				},
+				Memory: structs.AllocatedMemoryResources{
+					MemoryMB: 3200,
+				},
+				Networks: []*structs.NetworkResource{
+					{
+						Device:        "eth0",
+						IP:            "192.168.0.100",
+						ReservedPorts: []structs.Port{{Label: "admin", Value: 5000}},
+						MBits:         800,
+						DynamicPorts:  []structs.Port{{Label: "http", Value: 9876}},
+					},
+				},
+			},
+		},
+	}
+	plan := &structs.Plan{
+		Job: alloc.Job,
+		NodeAllocation: map[string][]*structs.Allocation{
+			node.ID: {alloc},
+		},
+		NodePreemptions: map[string][]*structs.Allocation{
+			node.ID: {preemptedAlloc},
+		},
+		Deployment: mock.Deployment(),
+		DeploymentUpdates: []*structs.DeploymentStatusUpdate{
+			{
+				DeploymentID:      uuid.Generate(),
+				Status:            "foo",
+				StatusDescription: "bar",
+			},
+		},
+	}
+	snap, _ := state.Snapshot()
+
+	pool := NewEvaluatePool(workerPoolSize, workerPoolBufferSize)
+	defer pool.Shutdown()
+
+	result, err := evaluatePlan(pool, snap, plan, testlog.HCLogger(t))
+
+	require := require.New(t)
+	require.NoError(err)
+	require.NotNil(result)
+
+	require.Equal(result.NodeAllocation, plan.NodeAllocation)
+	require.Equal(result.Deployment, plan.Deployment)
+	require.Equal(result.DeploymentUpdates, plan.DeploymentUpdates)
+	require.Equal(result.NodePreemptions, plan.NodePreemptions)
+
+}
+
 func TestPlanApply_EvalPlan_Partial(t *testing.T) {
 	t.Parallel()
 	state := testStateStore(t)

nomad/state/state_store.go

@@ -3889,9 +3889,9 @@ func (s *StateStore) SchedulerSetConfig(idx uint64, config *structs.SchedulerCon
 	return nil
 }
 
-// SchedulerCASConfig is used to try updating the scheduler configuration with a
+// SchedulerCASConfig is used to update the scheduler configuration with a
 // given Raft index. If the CAS index specified is not equal to the last observed index
-// for the config, then the call is a noop,
+// for the config, then the call is a noop.
 func (s *StateStore) SchedulerCASConfig(idx, cidx uint64, config *structs.SchedulerConfiguration) (bool, error) {
 	tx := s.db.Txn(true)
 	defer tx.Abort()

nomad/structs/structs.go

@@ -7411,37 +7411,6 @@ func (a *Allocation) ComparableResources() *ComparableResources {
 	}
 }
 
-// COMPAT(0.11): Remove in 0.11
-// CompatibleNetworkResources returns network resources on the allocation
-// by reading AllocatedResources which are populated starting in 0.9 and
-// falling back to pre 0.9 fields (Resources/TaskResources) if set
-func (a *Allocation) CompatibleNetworkResources() []*NetworkResource {
-	var ret []*NetworkResource
-	// Alloc already has 0.9+ behavior
-	if a.AllocatedResources != nil {
-		var comparableResources *ComparableResources
-		for _, taskResource := range a.AllocatedResources.Tasks {
-			if comparableResources == nil {
-				comparableResources = taskResource.Comparable()
-			} else {
-				comparableResources.Add(taskResource.Comparable())
-			}
-		}
-		ret = comparableResources.Flattened.Networks
-	} else if a.Resources != nil {
-		// Alloc has pre 0.9 total resources
-		ret = a.Resources.Networks
-	} else if a.TaskResources != nil {
-		// Alloc has pre 0.9 task resources
-		resources := new(Resources)
-		for _, taskResource := range a.TaskResources {
-			resources.Add(taskResource)
-		}
-		ret = resources.Networks
-	}
-	return ret
-}
-
 // LookupTask by name from the Allocation. Returns nil if the Job is not set, the
 // TaskGroup does not exist, or the task name cannot be found.
 func (a *Allocation) LookupTask(name string) *Task {
@@ -8207,6 +8176,7 @@ func (p *Plan) AppendPreemptedAlloc(alloc *Allocation, desiredStatus, preempting
 	if alloc.AllocatedResources != nil {
 		newAlloc.AllocatedResources = alloc.AllocatedResources
 	} else {
+		// COMPAT Remove in version 0.11
 		newAlloc.TaskResources = alloc.TaskResources
 		newAlloc.SharedResources = alloc.SharedResources
 	}