Merge pull request #4259 from hashicorp/f-deployment-improvements

api/allocations.go

@@ -143,6 +143,8 @@ type AllocationListStub struct {
 // healthy.
 type AllocDeploymentStatus struct {
 	Healthy     *bool
+	Timestamp   time.Time
+	Canary      bool
 	ModifyIndex uint64
 }
 
@@ -214,6 +216,10 @@ type DesiredTransition struct {
 	// Migrate is used to indicate that this allocation should be stopped and
 	// migrated to another node.
 	Migrate *bool
+
+	// Reschedule is used to indicate that this allocation is eligible to be
+	// rescheduled.
+	Reschedule *bool
 }
 
 // ShouldMigrate returns whether the transition object dictates a migration.

api/deployments.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"sort"
+	"time"
 )
 
 // Deployments is used to query the deployments endpoints.
@@ -139,14 +140,16 @@ type Deployment struct {
 
 // DeploymentState tracks the state of a deployment for a given task group.
 type DeploymentState struct {
-	PlacedCanaries  []string
-	AutoRevert      bool
-	Promoted        bool
-	DesiredCanaries int
-	DesiredTotal    int
-	PlacedAllocs    int
-	HealthyAllocs   int
-	UnhealthyAllocs int
+	PlacedCanaries    []string
+	AutoRevert        bool
+	ProgressDeadline  time.Duration
+	RequireProgressBy time.Time
+	Promoted          bool
+	DesiredCanaries   int
+	DesiredTotal      int
+	PlacedAllocs      int
+	HealthyAllocs     int
+	UnhealthyAllocs   int
 }
 
 // DeploymentIndexSort is a wrapper to sort deployments by CreateIndex. We

api/jobs.go

@@ -343,26 +343,28 @@ type periodicForceResponse struct {
 
 // UpdateStrategy defines a task groups update strategy.
 type UpdateStrategy struct {
-	Stagger         *time.Duration `mapstructure:"stagger"`
-	MaxParallel     *int           `mapstructure:"max_parallel"`
-	HealthCheck     *string        `mapstructure:"health_check"`
-	MinHealthyTime  *time.Duration `mapstructure:"min_healthy_time"`
-	HealthyDeadline *time.Duration `mapstructure:"healthy_deadline"`
-	AutoRevert      *bool          `mapstructure:"auto_revert"`
-	Canary          *int           `mapstructure:"canary"`
+	Stagger          *time.Duration `mapstructure:"stagger"`
+	MaxParallel      *int           `mapstructure:"max_parallel"`
+	HealthCheck      *string        `mapstructure:"health_check"`
+	MinHealthyTime   *time.Duration `mapstructure:"min_healthy_time"`
+	HealthyDeadline  *time.Duration `mapstructure:"healthy_deadline"`
+	ProgressDeadline *time.Duration `mapstructure:"progress_deadline"`
+	AutoRevert       *bool          `mapstructure:"auto_revert"`
+	Canary           *int           `mapstructure:"canary"`
 }
 
 // DefaultUpdateStrategy provides a baseline that can be used to upgrade
 // jobs with the old policy or for populating field defaults.
 func DefaultUpdateStrategy() *UpdateStrategy {
 	return &UpdateStrategy{
-		Stagger:         helper.TimeToPtr(30 * time.Second),
-		MaxParallel:     helper.IntToPtr(1),
-		HealthCheck:     helper.StringToPtr("checks"),
-		MinHealthyTime:  helper.TimeToPtr(10 * time.Second),
-		HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
-		AutoRevert:      helper.BoolToPtr(false),
-		Canary:          helper.IntToPtr(0),
+		Stagger:          helper.TimeToPtr(30 * time.Second),
+		MaxParallel:      helper.IntToPtr(1),
+		HealthCheck:      helper.StringToPtr("checks"),
+		MinHealthyTime:   helper.TimeToPtr(10 * time.Second),
+		HealthyDeadline:  helper.TimeToPtr(5 * time.Minute),
+		ProgressDeadline: helper.TimeToPtr(10 * time.Minute),
+		AutoRevert:       helper.BoolToPtr(false),
+		Canary:           helper.IntToPtr(0),
 	}
 }
 
@@ -393,6 +395,10 @@ func (u *UpdateStrategy) Copy() *UpdateStrategy {
 		copy.HealthyDeadline = helper.TimeToPtr(*u.HealthyDeadline)
 	}
 
+	if u.ProgressDeadline != nil {
+		copy.ProgressDeadline = helper.TimeToPtr(*u.ProgressDeadline)
+	}
+
 	if u.AutoRevert != nil {
 		copy.AutoRevert = helper.BoolToPtr(*u.AutoRevert)
 	}
@@ -429,6 +435,10 @@ func (u *UpdateStrategy) Merge(o *UpdateStrategy) {
 		u.HealthyDeadline = helper.TimeToPtr(*o.HealthyDeadline)
 	}
 
+	if o.ProgressDeadline != nil {
+		u.ProgressDeadline = helper.TimeToPtr(*o.ProgressDeadline)
+	}
+
 	if o.AutoRevert != nil {
 		u.AutoRevert = helper.BoolToPtr(*o.AutoRevert)
 	}
@@ -457,6 +467,10 @@ func (u *UpdateStrategy) Canonicalize() {
 		u.HealthyDeadline = d.HealthyDeadline
 	}
 
+	if u.ProgressDeadline == nil {
+		u.ProgressDeadline = d.ProgressDeadline
+	}
+
 	if u.MinHealthyTime == nil {
 		u.MinHealthyTime = d.MinHealthyTime
 	}
@@ -496,6 +510,10 @@ func (u *UpdateStrategy) Empty() bool {
 		return false
 	}
 
+	if u.ProgressDeadline != nil && *u.ProgressDeadline != 0 {
+		return false
+	}
+
 	if u.AutoRevert != nil && *u.AutoRevert {
 		return false
 	}

api/jobs_test.go

@@ -304,9 +304,10 @@ func TestJobs_Canonicalize(t *testing.T) {
 								},
 								Services: []*Service{
 									{
-										Name:      "redis-cache",
-										Tags:      []string{"global", "cache"},
-										PortLabel: "db",
+										Name:       "redis-cache",
+										Tags:       []string{"global", "cache"},
+										CanaryTags: []string{"canary", "global", "cache"},
+										PortLabel:  "db",
 										Checks: []ServiceCheck{
 											{
 												Name:     "alive",
@@ -354,13 +355,14 @@ func TestJobs_Canonicalize(t *testing.T) {
 				JobModifyIndex:    helper.Uint64ToPtr(0),
 				Datacenters:       []string{"dc1"},
 				Update: &UpdateStrategy{
-					Stagger:         helper.TimeToPtr(30 * time.Second),
-					MaxParallel:     helper.IntToPtr(1),
-					HealthCheck:     helper.StringToPtr("checks"),
-					MinHealthyTime:  helper.TimeToPtr(10 * time.Second),
-					HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
-					AutoRevert:      helper.BoolToPtr(false),
-					Canary:          helper.IntToPtr(0),
+					Stagger:          helper.TimeToPtr(30 * time.Second),
+					MaxParallel:      helper.IntToPtr(1),
+					HealthCheck:      helper.StringToPtr("checks"),
+					MinHealthyTime:   helper.TimeToPtr(10 * time.Second),
+					HealthyDeadline:  helper.TimeToPtr(5 * time.Minute),
+					ProgressDeadline: helper.TimeToPtr(10 * time.Minute),
+					AutoRevert:       helper.BoolToPtr(false),
+					Canary:           helper.IntToPtr(0),
 				},
 				TaskGroups: []*TaskGroup{
 					{
@@ -387,13 +389,14 @@ func TestJobs_Canonicalize(t *testing.T) {
 						},
 
 						Update: &UpdateStrategy{
-							Stagger:         helper.TimeToPtr(30 * time.Second),
-							MaxParallel:     helper.IntToPtr(1),
-							HealthCheck:     helper.StringToPtr("checks"),
-							MinHealthyTime:  helper.TimeToPtr(10 * time.Second),
-							HealthyDeadline: helper.TimeToPtr(5 * time.Minute),
-							AutoRevert:      helper.BoolToPtr(false),
-							Canary:          helper.IntToPtr(0),
+							Stagger:          helper.TimeToPtr(30 * time.Second),
+							MaxParallel:      helper.IntToPtr(1),
+							HealthCheck:      helper.StringToPtr("checks"),
+							MinHealthyTime:   helper.TimeToPtr(10 * time.Second),
+							HealthyDeadline:  helper.TimeToPtr(5 * time.Minute),
+							ProgressDeadline: helper.TimeToPtr(10 * time.Minute),
+							AutoRevert:       helper.BoolToPtr(false),
+							Canary:           helper.IntToPtr(0),
 						},
 						Migrate: DefaultMigrateStrategy(),
 						Tasks: []*Task{
@@ -425,6 +428,7 @@ func TestJobs_Canonicalize(t *testing.T) {
 									{
 										Name:        "redis-cache",
 										Tags:        []string{"global", "cache"},
+										CanaryTags:  []string{"canary", "global", "cache"},
 										PortLabel:   "db",
 										AddressMode: "auto",
 										Checks: []ServiceCheck{
@@ -515,13 +519,14 @@ func TestJobs_Canonicalize(t *testing.T) {
 				ID:       helper.StringToPtr("bar"),
 				ParentID: helper.StringToPtr("lol"),
 				Update: &UpdateStrategy{
-					Stagger:         helper.TimeToPtr(1 * time.Second),
-					MaxParallel:     helper.IntToPtr(1),
-					HealthCheck:     helper.StringToPtr("checks"),
-					MinHealthyTime:  helper.TimeToPtr(10 * time.Second),
-					HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
-					AutoRevert:      helper.BoolToPtr(false),
-					Canary:          helper.IntToPtr(0),
+					Stagger:          helper.TimeToPtr(1 * time.Second),
+					MaxParallel:      helper.IntToPtr(1),
+					HealthCheck:      helper.StringToPtr("checks"),
+					MinHealthyTime:   helper.TimeToPtr(10 * time.Second),
+					HealthyDeadline:  helper.TimeToPtr(6 * time.Minute),
+					ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
+					AutoRevert:       helper.BoolToPtr(false),
+					Canary:           helper.IntToPtr(0),
 				},
 				TaskGroups: []*TaskGroup{
 					{
@@ -569,13 +574,14 @@ func TestJobs_Canonicalize(t *testing.T) {
 				ModifyIndex:       helper.Uint64ToPtr(0),
 				JobModifyIndex:    helper.Uint64ToPtr(0),
 				Update: &UpdateStrategy{
-					Stagger:         helper.TimeToPtr(1 * time.Second),
-					MaxParallel:     helper.IntToPtr(1),
-					HealthCheck:     helper.StringToPtr("checks"),
-					MinHealthyTime:  helper.TimeToPtr(10 * time.Second),
-					HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
-					AutoRevert:      helper.BoolToPtr(false),
-					Canary:          helper.IntToPtr(0),
+					Stagger:          helper.TimeToPtr(1 * time.Second),
+					MaxParallel:      helper.IntToPtr(1),
+					HealthCheck:      helper.StringToPtr("checks"),
+					MinHealthyTime:   helper.TimeToPtr(10 * time.Second),
+					HealthyDeadline:  helper.TimeToPtr(6 * time.Minute),
+					ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
+					AutoRevert:       helper.BoolToPtr(false),
+					Canary:           helper.IntToPtr(0),
 				},
 				TaskGroups: []*TaskGroup{
 					{
@@ -601,13 +607,14 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Unlimited:     helper.BoolToPtr(true),
 						},
 						Update: &UpdateStrategy{
-							Stagger:         helper.TimeToPtr(2 * time.Second),
-							MaxParallel:     helper.IntToPtr(2),
-							HealthCheck:     helper.StringToPtr("manual"),
-							MinHealthyTime:  helper.TimeToPtr(1 * time.Second),
-							HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
-							AutoRevert:      helper.BoolToPtr(true),
-							Canary:          helper.IntToPtr(1),
+							Stagger:          helper.TimeToPtr(2 * time.Second),
+							MaxParallel:      helper.IntToPtr(2),
+							HealthCheck:      helper.StringToPtr("manual"),
+							MinHealthyTime:   helper.TimeToPtr(1 * time.Second),
+							HealthyDeadline:  helper.TimeToPtr(6 * time.Minute),
+							ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
+							AutoRevert:       helper.BoolToPtr(true),
+							Canary:           helper.IntToPtr(1),
 						},
 						Migrate: DefaultMigrateStrategy(),
 						Tasks: []*Task{
@@ -642,13 +649,14 @@ func TestJobs_Canonicalize(t *testing.T) {
 							Unlimited:     helper.BoolToPtr(true),
 						},
 						Update: &UpdateStrategy{
-							Stagger:         helper.TimeToPtr(1 * time.Second),
-							MaxParallel:     helper.IntToPtr(1),
-							HealthCheck:     helper.StringToPtr("checks"),
-							MinHealthyTime:  helper.TimeToPtr(10 * time.Second),
-							HealthyDeadline: helper.TimeToPtr(6 * time.Minute),
-							AutoRevert:      helper.BoolToPtr(false),
-							Canary:          helper.IntToPtr(0),
+							Stagger:          helper.TimeToPtr(1 * time.Second),
+							MaxParallel:      helper.IntToPtr(1),
+							HealthCheck:      helper.StringToPtr("checks"),
+							MinHealthyTime:   helper.TimeToPtr(10 * time.Second),
+							HealthyDeadline:  helper.TimeToPtr(6 * time.Minute),
+							ProgressDeadline: helper.TimeToPtr(7 * time.Minute),
+							AutoRevert:       helper.BoolToPtr(false),
+							Canary:           helper.IntToPtr(0),
 						},
 						Migrate: DefaultMigrateStrategy(),
 						Tasks: []*Task{

api/tasks.go

@@ -295,8 +295,9 @@ type Service struct {
 	Id           string
 	Name         string
 	Tags         []string
-	PortLabel    string `mapstructure:"port"`
-	AddressMode  string `mapstructure:"address_mode"`
+	CanaryTags   []string `mapstructure:"canary_tags"`
+	PortLabel    string   `mapstructure:"port"`
+	AddressMode  string   `mapstructure:"address_mode"`
 	Checks       []ServiceCheck
 	CheckRestart *CheckRestart `mapstructure:"check_restart"`
 }

api/tasks_test.go

@@ -252,13 +252,14 @@ func TestTaskGroup_Canonicalize_Update(t *testing.T) {
 	job := &Job{
 		ID: helper.StringToPtr("test"),
 		Update: &UpdateStrategy{
-			AutoRevert:      helper.BoolToPtr(false),
-			Canary:          helper.IntToPtr(0),
-			HealthCheck:     helper.StringToPtr(""),
-			HealthyDeadline: helper.TimeToPtr(0),
-			MaxParallel:     helper.IntToPtr(0),
-			MinHealthyTime:  helper.TimeToPtr(0),
-			Stagger:         helper.TimeToPtr(0),
+			AutoRevert:       helper.BoolToPtr(false),
+			Canary:           helper.IntToPtr(0),
+			HealthCheck:      helper.StringToPtr(""),
+			HealthyDeadline:  helper.TimeToPtr(0),
+			ProgressDeadline: helper.TimeToPtr(0),
+			MaxParallel:      helper.IntToPtr(0),
+			MinHealthyTime:   helper.TimeToPtr(0),
+			Stagger:          helper.TimeToPtr(0),
 		},
 	}
 	job.Canonicalize()

client/alloc_runner.go

@@ -50,7 +50,8 @@ type AllocRunner struct {
 	alloc                  *structs.Allocation
 	allocClientStatus      string // Explicit status of allocation. Set when there are failures
 	allocClientDescription string
-	allocHealth            *bool // Whether the allocation is healthy
+	allocHealth            *bool     // Whether the allocation is healthy
+	allocHealthTime        time.Time // Time at which allocation health has been set
 	allocBroadcast         *cstructs.AllocBroadcaster
 	allocLock              sync.Mutex
 
@@ -580,6 +581,7 @@ func (r *AllocRunner) Alloc() *structs.Allocation {
 			alloc.DeploymentStatus = &structs.AllocDeploymentStatus{}
 		}
 		alloc.DeploymentStatus.Healthy = helper.BoolToPtr(*r.allocHealth)
+		alloc.DeploymentStatus.Timestamp = r.allocHealthTime
 	}
 	r.allocLock.Unlock()
 
@@ -943,6 +945,7 @@ OUTER:
 			// If the deployment ids have changed clear the health
 			if r.alloc.DeploymentID != update.DeploymentID {
 				r.allocHealth = nil
+				r.allocHealthTime = time.Time{}
 			}
 
 			r.alloc = update

client/alloc_runner_health_watcher.go

@@ -112,6 +112,7 @@ func (r *AllocRunner) watchHealth(ctx context.Context) {
 
 	r.allocLock.Lock()
 	r.allocHealth = helper.BoolToPtr(allocHealthy)
+	r.allocHealthTime = time.Now()
 	r.allocLock.Unlock()
 
 	// If deployment is unhealthy emit task events explaining why

client/consul.go

@@ -1,17 +1,14 @@
 package client
 
 import (
-	"github.com/hashicorp/nomad/client/driver"
-	cstructs "github.com/hashicorp/nomad/client/structs"
 	"github.com/hashicorp/nomad/command/agent/consul"
-	"github.com/hashicorp/nomad/nomad/structs"
 )
 
 // ConsulServiceAPI is the interface the Nomad Client uses to register and
 // remove services and checks from Consul.
 type ConsulServiceAPI interface {
-	RegisterTask(allocID string, task *structs.Task, restarter consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error
-	RemoveTask(allocID string, task *structs.Task)
-	UpdateTask(allocID string, existing, newTask *structs.Task, restart consul.TaskRestarter, exec driver.ScriptExecutor, net *cstructs.DriverNetwork) error
+	RegisterTask(*consul.TaskServices) error
+	RemoveTask(*consul.TaskServices)
+	UpdateTask(old, newTask *consul.TaskServices) error
 	AllocRegistrations(allocID string) (*consul.AllocRegistration, error)
 }