Changing the state of an allocation to lost if the node on which it w…

…as running was marked as down

nomad/state/state_store.go

@@ -192,6 +192,24 @@ func (s *StateStore) UpdateNodeStatus(index uint64, nodeID, status string) error
 		return fmt.Errorf("index update failed: %v", err)
 	}
 
+	// Update the state of the allocations which are in running state to lost
+	if status == structs.NodeStatusDown {
+		allocs, err := s.AllocsByNode(nodeID)
+		if err != nil {
+			return fmt.Errorf("error retrieving any allocations for the node: %v", nodeID)
+		}
+		for _, alloc := range allocs {
+			copyAlloc := new(structs.Allocation)
+			*copyAlloc = *alloc
+			if alloc.ClientStatus == structs.AllocClientStatusRunning {
+				copyAlloc.ClientStatus = structs.AllocClientStatusLost
+				if err := txn.Insert("allocs", copyAlloc); err != nil {
+					return fmt.Errorf("alloc insert failed: %v", err)
+				}
+			}
+		}
+	}
+
 	txn.Defer(func() { s.watch.notify(watcher) })
 	txn.Commit()
 	return nil

nomad/state/state_store_test.go

@@ -136,6 +136,36 @@ func TestStateStore_UpdateNodeStatus_Node(t *testing.T) {
 		t.Fatalf("bad: %d", index)
 	}
 
+	alloc := mock.Alloc()
+	alloc1 := mock.Alloc()
+	alloc.NodeID = node.ID
+	alloc1.NodeID = node.ID
+	alloc.ClientStatus = structs.AllocClientStatusRunning
+	alloc1.ClientStatus = structs.AllocClientStatusFailed
+
+	if err = state.UpsertAllocs(1002, []*structs.Allocation{alloc, alloc1}); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if err = state.UpdateNodeStatus(1003, node.ID, structs.NodeStatusDown); err != nil {
+		t.Fatalf("err: %v", err)
+	}
+
+	allocOut, err := state.AllocByID(alloc.ID)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if allocOut.ClientStatus != structs.AllocClientStatusLost {
+		t.Fatalf("expected alloc status: %v, actual: %v", structs.AllocClientStatusLost, allocOut.ClientStatus)
+	}
+
+	alloc1Out, err := state.AllocByID(alloc1.ID)
+	if err != nil {
+		t.Fatalf("err: %v", err)
+	}
+	if alloc1Out.ClientStatus != structs.AllocClientStatusFailed {
+		t.Fatalf("expected alloc status: %v, actual: %v", structs.AllocClientStatusLost, allocOut.ClientStatus)
+	}
+
 	notify.verify(t)
 }
 

nomad/structs/structs.go

@@ -2304,6 +2304,7 @@ const (
 	AllocClientStatusRunning  = "running"
 	AllocClientStatusComplete = "complete"
 	AllocClientStatusFailed   = "failed"
+	AllocClientStatusLost     = "lost"
 )
 
 // Allocation is used to allocate the placement of a task group to a node.
@@ -2410,7 +2411,7 @@ func (a *Allocation) TerminalStatus() bool {
 	}
 
 	switch a.ClientStatus {
-	case AllocClientStatusComplete, AllocClientStatusFailed:
+	case AllocClientStatusComplete, AllocClientStatusFailed, AllocClientStatusLost:
 		return true
 	default:
 		return false